2026-02-22T22:32:49.390 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-22T22:32:49.408 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:32:49.489 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259 branch: wip-yuri6-testing-2026-02-22-1629-squid description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-bootstrap-workunit} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '64259' kernel: branch: distro kdb: true sha1: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run the rbd_mirror_bootstrap.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-yuri6-testing-2026-02-22-1629-squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bdev async discard: true bdev enable discard: true bluestore allocator: stupid bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 ceph-deploy: bluestore: true conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd objectstore: bluestore fs: xfs install: ceph: flavor: default sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 selinux: allowlist: - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri6-testing-2026-02-22-1629-squid sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 9196 sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-yuri6-testing-2026-02-22-1629-squid suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 targets: trial078.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDNDWZiJW4+mOO5x9eBXVo1SH1y+4ZVaepjXMZrz5ZUr76fZJVBweiNldE6oG8JquHGeOG7ACQ7DVVHDO5xMleY= trial178.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMgjfWFN++E60nfU1X/ZHTxvvvI+OpzCzJpV7wLGORR/Gntf4Ene1CfaJ7Lliu5GSNruuGy7XXe+1TU5E4tsGbg= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_bootstrap.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: image RBD_MIRROR_INSTANCES: '1' RBD_MIRROR_MODE: snapshot RBD_MIRROR_USE_EXISTING_CLUSTER: '1' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-22_19:35:38 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653 2026-02-22T22:32:49.490 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa; will attempt to use it 2026-02-22T22:32:49.490 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa/tasks 2026-02-22T22:32:49.490 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-22T22:32:49.492 INFO:teuthology.task.internal:Checking packages... 2026-02-22T22:32:49.492 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f90f4eaebb28b619cf76e80cbbe997656da69579' 2026-02-22T22:32:49.492 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-22T22:32:49.492 INFO:teuthology.packaging:ref: None 2026-02-22T22:32:49.492 INFO:teuthology.packaging:tag: None 2026-02-22T22:32:49.492 INFO:teuthology.packaging:branch: wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:32:49.492 INFO:teuthology.packaging:sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:32:49.492 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:32:49.613 INFO:teuthology.task.internal:Found packages for ceph version 19.2.3-689.gf90f4eae 2026-02-22T22:32:49.614 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-22T22:32:49.626 INFO:teuthology.task.internal:no buildpackages task found 2026-02-22T22:32:49.626 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-22T22:32:49.639 INFO:teuthology.task.internal:Saving configuration 2026-02-22T22:32:49.655 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-22T22:32:49.667 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-22T22:32:49.743 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial078.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 22:29:18.717485', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDNDWZiJW4+mOO5x9eBXVo1SH1y+4ZVaepjXMZrz5ZUr76fZJVBweiNldE6oG8JquHGeOG7ACQ7DVVHDO5xMleY='} 2026-02-22T22:32:49.798 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial178.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 22:29:18.716850', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMgjfWFN++E60nfU1X/ZHTxvvvI+OpzCzJpV7wLGORR/Gntf4Ene1CfaJ7Lliu5GSNruuGy7XXe+1TU5E4tsGbg='} 2026-02-22T22:32:49.798 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-22T22:32:49.808 INFO:teuthology.task.internal:roles: ubuntu@trial078.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-22T22:32:49.808 INFO:teuthology.task.internal:roles: ubuntu@trial178.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-22T22:32:49.809 INFO:teuthology.run_tasks:Running task console_log... 2026-02-22T22:32:49.939 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7e4b162f2f20>, signals=[15]) 2026-02-22T22:32:49.939 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-22T22:32:49.949 INFO:teuthology.task.internal:Opening connections... 2026-02-22T22:32:49.949 DEBUG:teuthology.task.internal:connecting to ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:32:49.950 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:32:50.011 DEBUG:teuthology.task.internal:connecting to ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:32:50.011 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial178.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:32:50.078 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-22T22:32:50.090 DEBUG:teuthology.orchestra.run.trial078:> uname -m 2026-02-22T22:32:50.105 INFO:teuthology.orchestra.run.trial078.stdout:x86_64 2026-02-22T22:32:50.105 DEBUG:teuthology.orchestra.run.trial078:> cat /etc/os-release 2026-02-22T22:32:50.161 INFO:teuthology.orchestra.run.trial078.stdout:NAME="CentOS Stream" 2026-02-22T22:32:50.161 INFO:teuthology.orchestra.run.trial078.stdout:VERSION="9" 2026-02-22T22:32:50.161 INFO:teuthology.orchestra.run.trial078.stdout:ID="centos" 2026-02-22T22:32:50.161 INFO:teuthology.orchestra.run.trial078.stdout:ID_LIKE="rhel fedora" 2026-02-22T22:32:50.161 INFO:teuthology.orchestra.run.trial078.stdout:VERSION_ID="9" 2026-02-22T22:32:50.161 INFO:teuthology.orchestra.run.trial078.stdout:PLATFORM_ID="platform:el9" 2026-02-22T22:32:50.161 INFO:teuthology.orchestra.run.trial078.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T22:32:50.162 INFO:teuthology.orchestra.run.trial078.stdout:ANSI_COLOR="0;31" 2026-02-22T22:32:50.162 INFO:teuthology.orchestra.run.trial078.stdout:LOGO="fedora-logo-icon" 2026-02-22T22:32:50.162 INFO:teuthology.orchestra.run.trial078.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T22:32:50.162 INFO:teuthology.orchestra.run.trial078.stdout:HOME_URL="https://centos.org/" 2026-02-22T22:32:50.162 INFO:teuthology.orchestra.run.trial078.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T22:32:50.162 INFO:teuthology.orchestra.run.trial078.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T22:32:50.162 INFO:teuthology.orchestra.run.trial078.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T22:32:50.162 INFO:teuthology.lock.ops:Updating trial078.front.sepia.ceph.com on lock server 2026-02-22T22:32:50.235 DEBUG:teuthology.orchestra.run.trial178:> uname -m 2026-02-22T22:32:50.250 INFO:teuthology.orchestra.run.trial178.stdout:x86_64 2026-02-22T22:32:50.250 DEBUG:teuthology.orchestra.run.trial178:> cat /etc/os-release 2026-02-22T22:32:50.306 INFO:teuthology.orchestra.run.trial178.stdout:NAME="CentOS Stream" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:VERSION="9" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:ID="centos" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:ID_LIKE="rhel fedora" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:VERSION_ID="9" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:PLATFORM_ID="platform:el9" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:ANSI_COLOR="0;31" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:LOGO="fedora-logo-icon" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:HOME_URL="https://centos.org/" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T22:32:50.307 INFO:teuthology.orchestra.run.trial178.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T22:32:50.307 INFO:teuthology.lock.ops:Updating trial178.front.sepia.ceph.com on lock server 2026-02-22T22:32:50.865 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-22T22:32:50.881 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-22T22:32:50.893 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-22T22:32:50.893 DEBUG:teuthology.orchestra.run.trial078:> test '!' -e /home/ubuntu/cephtest 2026-02-22T22:32:50.896 DEBUG:teuthology.orchestra.run.trial178:> test '!' -e /home/ubuntu/cephtest 2026-02-22T22:32:50.912 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-22T22:32:50.923 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-22T22:32:50.923 DEBUG:teuthology.orchestra.run.trial078:> test -z $(ls -A /var/lib/ceph) 2026-02-22T22:32:50.951 DEBUG:teuthology.orchestra.run.trial178:> test -z $(ls -A /var/lib/ceph) 2026-02-22T22:32:50.968 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-22T22:32:51.149 INFO:teuthology.run_tasks:Running task kernel... 2026-02-22T22:32:51.168 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-22T22:32:51.168 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-22T22:32:51.169 DEBUG:teuthology.orchestra.run.trial078:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T22:32:51.169 DEBUG:teuthology.orchestra.run.trial178:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T22:32:51.183 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:32:51.184 DEBUG:teuthology.orchestra.run.trial078:> uname -r 2026-02-22T22:32:51.184 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:32:51.184 DEBUG:teuthology.orchestra.run.trial178:> uname -r 2026-02-22T22:32:51.239 INFO:teuthology.orchestra.run.trial078.stdout:5.14.0-681.el9.x86_64 2026-02-22T22:32:51.239 INFO:teuthology.task.kernel:Running kernel on trial078: 5.14.0-681.el9.x86_64 2026-02-22T22:32:51.239 DEBUG:teuthology.orchestra.run.trial078:> sudo yum install -y kernel 2026-02-22T22:32:51.240 INFO:teuthology.orchestra.run.trial178.stdout:5.14.0-681.el9.x86_64 2026-02-22T22:32:51.240 INFO:teuthology.task.kernel:Running kernel on trial178: 5.14.0-681.el9.x86_64 2026-02-22T22:32:51.240 DEBUG:teuthology.orchestra.run.trial178:> sudo yum install -y kernel 2026-02-22T22:32:52.321 INFO:teuthology.orchestra.run.trial078.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.9 MB 00:00 2026-02-22T22:32:52.477 INFO:teuthology.orchestra.run.trial178.stdout:CentOS Stream 9 - BaseOS 10 MB/s | 8.9 MB 00:00 2026-02-22T22:32:55.903 INFO:teuthology.orchestra.run.trial178.stdout:CentOS Stream 9 - AppStream 9.2 MB/s | 27 MB 00:02 2026-02-22T22:32:56.040 INFO:teuthology.orchestra.run.trial078.stdout:CentOS Stream 9 - AppStream 8.4 MB/s | 27 MB 00:03 2026-02-22T22:32:59.459 INFO:teuthology.orchestra.run.trial078.stdout:CentOS Stream 9 - CRB 7.2 MB/s | 7.8 MB 00:01 2026-02-22T22:33:00.279 INFO:teuthology.orchestra.run.trial178.stdout:CentOS Stream 9 - CRB 3.8 MB/s | 7.8 MB 00:02 2026-02-22T22:33:00.368 INFO:teuthology.orchestra.run.trial078.stdout:CentOS Stream 9 - Extras packages 106 kB/s | 20 kB 00:00 2026-02-22T22:33:00.987 INFO:teuthology.orchestra.run.trial078.stdout:Extra Packages for Enterprise Linux 37 MB/s | 20 MB 00:00 2026-02-22T22:33:01.110 INFO:teuthology.orchestra.run.trial178.stdout:CentOS Stream 9 - Extras packages 174 kB/s | 20 kB 00:00 2026-02-22T22:33:01.755 INFO:teuthology.orchestra.run.trial178.stdout:Extra Packages for Enterprise Linux 35 MB/s | 20 MB 00:00 2026-02-22T22:33:04.218 INFO:teuthology.orchestra.run.trial078.stdout:lab-extras 2.0 MB/s | 50 kB 00:00 2026-02-22T22:33:04.927 INFO:teuthology.orchestra.run.trial178.stdout:lab-extras 2.7 MB/s | 50 kB 00:00 2026-02-22T22:33:05.234 INFO:teuthology.orchestra.run.trial078.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T22:33:05.234 INFO:teuthology.orchestra.run.trial078.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T22:33:05.253 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:33:05.253 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:33:05.253 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:33:05.278 DEBUG:teuthology.orchestra.run.trial078:> echo no | sudo yum reinstall kernel || true 2026-02-22T22:33:05.580 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:01 ago on Sun 22 Feb 2026 10:33:04 PM UTC. 2026-02-22T22:33:05.660 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout: Package Architecture Version Repository Size 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout:Reinstalling: 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 803 k 2026-02-22T22:33:05.661 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 0 2026-02-22T22:33:05.662 INFO:teuthology.orchestra.run.trial078.stderr:Operation aborted. 2026-02-22T22:33:05.681 INFO:teuthology.orchestra.run.trial078.stdout:Is this ok [y/N]: 2026-02-22T22:33:05.682 DEBUG:teuthology.orchestra.run.trial078:> sudo yum reinstall -y kernel || true 2026-02-22T22:33:05.937 INFO:teuthology.orchestra.run.trial178.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T22:33:05.938 INFO:teuthology.orchestra.run.trial178.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T22:33:05.957 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:33:05.957 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:33:05.957 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:33:05.982 DEBUG:teuthology.orchestra.run.trial178:> echo no | sudo yum reinstall kernel || true 2026-02-22T22:33:05.983 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:01 ago on Sun 22 Feb 2026 10:33:04 PM UTC. 2026-02-22T22:33:06.063 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:33:06.064 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:33:06.064 INFO:teuthology.orchestra.run.trial078.stdout: Package Architecture Version Repository Size 2026-02-22T22:33:06.064 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:33:06.064 INFO:teuthology.orchestra.run.trial078.stdout:Reinstalling: 2026-02-22T22:33:06.064 INFO:teuthology.orchestra.run.trial078.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:33:06.065 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:33:06.065 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:33:06.065 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:33:06.065 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:33:06.065 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 803 k 2026-02-22T22:33:06.065 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 0 2026-02-22T22:33:06.066 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:33:06.287 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:33:04 PM UTC. 2026-02-22T22:33:06.367 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout: Package Architecture Version Repository Size 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout:Reinstalling: 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:33:06.368 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:33:06.369 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 803 k 2026-02-22T22:33:06.369 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 0 2026-02-22T22:33:06.369 INFO:teuthology.orchestra.run.trial178.stderr:Operation aborted. 2026-02-22T22:33:06.388 INFO:teuthology.orchestra.run.trial178.stdout:Is this ok [y/N]: 2026-02-22T22:33:06.389 DEBUG:teuthology.orchestra.run.trial178:> sudo yum reinstall -y kernel || true 2026-02-22T22:33:06.691 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:33:04 PM UTC. 2026-02-22T22:33:06.768 INFO:teuthology.orchestra.run.trial078.stdout:kernel-5.14.0-681.el9.x86_64.rpm 1.2 MB/s | 803 kB 00:00 2026-02-22T22:33:06.768 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:33:06.768 INFO:teuthology.orchestra.run.trial078.stdout:Total 1.1 MB/s | 803 kB 00:00 2026-02-22T22:33:06.771 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout: Package Architecture Version Repository Size 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout:Reinstalling: 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:33:06.772 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:33:06.773 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 803 k 2026-02-22T22:33:06.773 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 0 2026-02-22T22:33:06.773 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:33:06.778 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:33:06.811 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:33:06.811 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:33:06.815 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:33:06.815 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:33:06.839 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:33:06.843 INFO:teuthology.orchestra.run.trial078.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:33:06.843 INFO:teuthology.orchestra.run.trial078.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:33:06.950 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:33:06.950 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:33:06.967 INFO:teuthology.orchestra.run.trial178.stdout:kernel-5.14.0-681.el9.x86_64.rpm 5.9 MB/s | 803 kB 00:00 2026-02-22T22:33:06.967 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:33:06.967 INFO:teuthology.orchestra.run.trial178.stdout:Total 4.0 MB/s | 803 kB 00:00 2026-02-22T22:33:06.977 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:33:07.007 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:33:07.007 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:33:07.007 INFO:teuthology.orchestra.run.trial078.stdout:Reinstalled: 2026-02-22T22:33:07.007 INFO:teuthology.orchestra.run.trial078.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T22:33:07.007 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:33:07.008 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:33:07.011 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:33:07.011 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:33:07.016 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:33:07.016 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:33:07.033 DEBUG:teuthology.orchestra.run.trial078:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T22:33:07.040 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:33:07.044 INFO:teuthology.orchestra.run.trial178.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:33:07.044 INFO:teuthology.orchestra.run.trial178.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:33:07.097 INFO:teuthology.orchestra.run.trial078.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T22:33:07.097 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T22:33:07.097 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T22:33:07.097 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-22T22:33:07.098 DEBUG:teuthology.orchestra.run.trial078:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T22:33:07.155 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:33:07.156 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:33:07.162 INFO:teuthology.orchestra.run.trial078.stdout:ttyS1 2026-02-22T22:33:07.184 DEBUG:teuthology.parallel:result is None 2026-02-22T22:33:07.213 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:33:07.213 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:33:07.213 INFO:teuthology.orchestra.run.trial178.stdout:Reinstalled: 2026-02-22T22:33:07.213 INFO:teuthology.orchestra.run.trial178.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T22:33:07.213 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:33:07.213 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:33:07.240 DEBUG:teuthology.orchestra.run.trial178:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T22:33:07.264 INFO:teuthology.orchestra.run.trial178.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T22:33:07.265 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T22:33:07.265 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T22:33:07.266 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-22T22:33:07.266 DEBUG:teuthology.orchestra.run.trial178:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T22:33:07.329 INFO:teuthology.orchestra.run.trial178.stdout:ttyS1 2026-02-22T22:33:07.361 DEBUG:teuthology.parallel:result is None 2026-02-22T22:33:07.361 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-22T22:33:07.373 INFO:teuthology.task.internal:Creating test directory... 2026-02-22T22:33:07.373 DEBUG:teuthology.orchestra.run.trial078:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T22:33:07.376 DEBUG:teuthology.orchestra.run.trial178:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T22:33:07.418 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-22T22:33:07.434 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-22T22:33:07.448 INFO:teuthology.task.internal:Creating archive directory... 2026-02-22T22:33:07.449 DEBUG:teuthology.orchestra.run.trial078:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T22:33:07.452 DEBUG:teuthology.orchestra.run.trial178:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T22:33:07.483 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-22T22:33:07.493 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-22T22:33:07.494 DEBUG:teuthology.orchestra.run.trial078:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T22:33:07.507 DEBUG:teuthology.orchestra.run.trial178:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T22:33:07.529 INFO:teuthology.orchestra.run.trial078.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:33:07.539 INFO:teuthology.orchestra.run.trial078.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:33:07.539 INFO:teuthology.orchestra.run.trial178.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:33:07.548 INFO:teuthology.orchestra.run.trial178.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:33:07.549 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-22T22:33:07.564 INFO:teuthology.task.internal:Configuring sudo... 2026-02-22T22:33:07.564 DEBUG:teuthology.orchestra.run.trial078:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T22:33:07.582 DEBUG:teuthology.orchestra.run.trial178:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T22:33:07.614 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-22T22:33:07.632 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-22T22:33:07.633 DEBUG:teuthology.orchestra.run.trial078:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T22:33:07.648 DEBUG:teuthology.orchestra.run.trial178:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T22:33:07.669 DEBUG:teuthology.orchestra.run.trial078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:33:07.850 DEBUG:teuthology.orchestra.run.trial078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:33:07.876 DEBUG:teuthology.orchestra.run.trial078:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:33:08.019 DEBUG:teuthology.orchestra.run.trial078:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:33:08.041 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:33:08.041 DEBUG:teuthology.orchestra.run.trial078:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T22:33:08.104 DEBUG:teuthology.orchestra.run.trial178:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:33:08.682 DEBUG:teuthology.orchestra.run.trial178:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:33:08.707 DEBUG:teuthology.orchestra.run.trial178:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:33:08.868 DEBUG:teuthology.orchestra.run.trial178:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:33:08.892 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:33:08.892 DEBUG:teuthology.orchestra.run.trial178:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T22:33:08.956 DEBUG:teuthology.orchestra.run.trial078:> sudo service rsyslog restart 2026-02-22T22:33:08.958 DEBUG:teuthology.orchestra.run.trial178:> sudo service rsyslog restart 2026-02-22T22:33:08.984 INFO:teuthology.orchestra.run.trial078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:33:09.024 INFO:teuthology.orchestra.run.trial178.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:33:09.433 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-22T22:33:09.449 INFO:teuthology.task.internal:Starting timer... 2026-02-22T22:33:09.449 INFO:teuthology.run_tasks:Running task pcp... 2026-02-22T22:33:09.471 INFO:teuthology.run_tasks:Running task selinux... 2026-02-22T22:33:09.489 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:getty_t:s0']} 2026-02-22T22:33:09.489 DEBUG:teuthology.orchestra.run.trial078:> sudo service auditd rotate 2026-02-22T22:33:09.517 INFO:teuthology.orchestra.run.trial078.stdout:Rotating logs: 2026-02-22T22:33:09.519 DEBUG:teuthology.orchestra.run.trial178:> sudo service auditd rotate 2026-02-22T22:33:09.548 INFO:teuthology.orchestra.run.trial178.stdout:Rotating logs: 2026-02-22T22:33:09.550 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-22T22:33:09.550 DEBUG:teuthology.orchestra.run.trial078:> /usr/sbin/getenforce 2026-02-22T22:33:09.576 INFO:teuthology.orchestra.run.trial078.stdout:Permissive 2026-02-22T22:33:09.576 DEBUG:teuthology.orchestra.run.trial178:> /usr/sbin/getenforce 2026-02-22T22:33:09.606 INFO:teuthology.orchestra.run.trial178.stdout:Permissive 2026-02-22T22:33:09.606 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@trial078.front.sepia.ceph.com': 'permissive', 'ubuntu@trial178.front.sepia.ceph.com': 'permissive'} 2026-02-22T22:33:09.607 DEBUG:teuthology.orchestra.run.trial078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:33:09.641 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:33:09.641 DEBUG:teuthology.orchestra.run.trial178:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:33:09.671 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:33:09.671 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-22T22:33:09.672 DEBUG:teuthology.orchestra.run.trial078:> sudo /usr/sbin/setenforce permissive 2026-02-22T22:33:09.705 DEBUG:teuthology.orchestra.run.trial178:> sudo /usr/sbin/setenforce permissive 2026-02-22T22:33:09.734 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-22T22:33:09.748 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-22T22:33:09.756 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-22T22:33:09.757 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial078.front.sepia.ceph.com,trial178.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-22T22:36:01.005 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial078.front.sepia.ceph.com'), Remote(name='ubuntu@trial178.front.sepia.ceph.com')] 2026-02-22T22:36:01.006 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial078.front.sepia.ceph.com' 2026-02-22T22:36:01.007 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial078.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:36:01.131 DEBUG:teuthology.orchestra.run.trial078:> true 2026-02-22T22:36:01.206 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial078.front.sepia.ceph.com' 2026-02-22T22:36:01.207 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial178.front.sepia.ceph.com' 2026-02-22T22:36:01.207 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial178.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:36:01.295 DEBUG:teuthology.orchestra.run.trial178:> true 2026-02-22T22:36:01.367 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial178.front.sepia.ceph.com' 2026-02-22T22:36:01.367 INFO:teuthology.run_tasks:Running task clock... 2026-02-22T22:36:01.379 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-22T22:36:01.379 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T22:36:01.380 DEBUG:teuthology.orchestra.run.trial078:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:36:01.382 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T22:36:01.382 DEBUG:teuthology.orchestra.run.trial178:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:36:01.408 INFO:teuthology.orchestra.run.trial078.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T22:36:01.420 INFO:teuthology.orchestra.run.trial078.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T22:36:01.435 INFO:teuthology.orchestra.run.trial178.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T22:36:01.440 INFO:teuthology.orchestra.run.trial078.stderr:sudo: ntpd: command not found 2026-02-22T22:36:01.447 INFO:teuthology.orchestra.run.trial178.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T22:36:01.450 INFO:teuthology.orchestra.run.trial078.stdout:506 Cannot talk to daemon 2026-02-22T22:36:01.462 INFO:teuthology.orchestra.run.trial078.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T22:36:01.469 INFO:teuthology.orchestra.run.trial178.stderr:sudo: ntpd: command not found 2026-02-22T22:36:01.474 INFO:teuthology.orchestra.run.trial078.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T22:36:01.479 INFO:teuthology.orchestra.run.trial178.stdout:506 Cannot talk to daemon 2026-02-22T22:36:01.490 INFO:teuthology.orchestra.run.trial178.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T22:36:01.502 INFO:teuthology.orchestra.run.trial178.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T22:36:01.521 INFO:teuthology.orchestra.run.trial078.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:36:01.522 INFO:teuthology.orchestra.run.trial078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:36:01.522 INFO:teuthology.orchestra.run.trial078.stdout:=============================================================================== 2026-02-22T22:36:01.551 INFO:teuthology.orchestra.run.trial178.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:36:01.552 INFO:teuthology.orchestra.run.trial178.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:36:01.552 INFO:teuthology.orchestra.run.trial178.stdout:=============================================================================== 2026-02-22T22:36:01.553 INFO:teuthology.run_tasks:Running task install... 2026-02-22T22:36:01.564 DEBUG:teuthology.task.install:project ceph 2026-02-22T22:36:01.564 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}} 2026-02-22T22:36:01.564 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'} 2026-02-22T22:36:01.564 INFO:teuthology.task.install:Using flavor: default 2026-02-22T22:36:01.567 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-22T22:36:01.567 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-22T22:36:01.567 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T22:36:01.567 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:36:01.568 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T22:36:01.568 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:36:01.749 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T22:36:01.749 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T22:36:01.760 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T22:36:01.760 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T22:36:01.898 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T22:36:01.898 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:36:01.898 DEBUG:teuthology.orchestra.run.trial078:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T22:36:01.908 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T22:36:01.909 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:36:01.909 DEBUG:teuthology.orchestra.run.trial178:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T22:36:01.923 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T22:36:01.924 DEBUG:teuthology.orchestra.run.trial078:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T22:36:01.934 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T22:36:01.934 DEBUG:teuthology.orchestra.run.trial178:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T22:36:01.988 DEBUG:teuthology.orchestra.run.trial078:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T22:36:01.998 DEBUG:teuthology.orchestra.run.trial178:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T22:36:02.064 DEBUG:teuthology.orchestra.run.trial078:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T22:36:02.071 DEBUG:teuthology.orchestra.run.trial178:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T22:36:02.129 INFO:teuthology.orchestra.run.trial078.stdout:check_obsoletes = 1 2026-02-22T22:36:02.130 DEBUG:teuthology.orchestra.run.trial078:> sudo yum clean all 2026-02-22T22:36:02.134 INFO:teuthology.orchestra.run.trial178.stdout:check_obsoletes = 1 2026-02-22T22:36:02.136 DEBUG:teuthology.orchestra.run.trial178:> sudo yum clean all 2026-02-22T22:36:02.291 INFO:teuthology.orchestra.run.trial078.stdout:41 files removed 2026-02-22T22:36:02.291 INFO:teuthology.orchestra.run.trial178.stdout:41 files removed 2026-02-22T22:36:02.305 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-radosgw 2026-02-22T22:36:02.307 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-radosgw 2026-02-22T22:36:02.587 INFO:teuthology.orchestra.run.trial078.stdout:ceph packages for x86_64 626 kB/s | 86 kB 00:00 2026-02-22T22:36:02.592 INFO:teuthology.orchestra.run.trial178.stdout:ceph packages for x86_64 599 kB/s | 86 kB 00:00 2026-02-22T22:36:02.732 INFO:teuthology.orchestra.run.trial178.stdout:ceph noarch packages 104 kB/s | 12 kB 00:00 2026-02-22T22:36:02.738 INFO:teuthology.orchestra.run.trial078.stdout:ceph noarch packages 95 kB/s | 12 kB 00:00 2026-02-22T22:36:02.857 INFO:teuthology.orchestra.run.trial178.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T22:36:02.866 INFO:teuthology.orchestra.run.trial078.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T22:36:03.554 INFO:teuthology.orchestra.run.trial178.stdout:CentOS Stream 9 - BaseOS 13 MB/s | 8.9 MB 00:00 2026-02-22T22:36:03.798 INFO:teuthology.orchestra.run.trial078.stdout:CentOS Stream 9 - BaseOS 9.7 MB/s | 8.9 MB 00:00 2026-02-22T22:36:06.438 INFO:teuthology.orchestra.run.trial178.stdout:CentOS Stream 9 - AppStream 11 MB/s | 27 MB 00:02 2026-02-22T22:36:07.008 INFO:teuthology.orchestra.run.trial078.stdout:CentOS Stream 9 - AppStream 10 MB/s | 27 MB 00:02 2026-02-22T22:36:10.290 INFO:teuthology.orchestra.run.trial078.stdout:CentOS Stream 9 - CRB 8.5 MB/s | 7.8 MB 00:00 2026-02-22T22:36:10.572 INFO:teuthology.orchestra.run.trial178.stdout:CentOS Stream 9 - CRB 4.4 MB/s | 7.8 MB 00:01 2026-02-22T22:36:11.281 INFO:teuthology.orchestra.run.trial078.stdout:CentOS Stream 9 - Extras packages 77 kB/s | 20 kB 00:00 2026-02-22T22:36:11.762 INFO:teuthology.orchestra.run.trial178.stdout:CentOS Stream 9 - Extras packages 43 kB/s | 20 kB 00:00 2026-02-22T22:36:11.865 INFO:teuthology.orchestra.run.trial078.stdout:Extra Packages for Enterprise Linux 39 MB/s | 20 MB 00:00 2026-02-22T22:36:12.750 INFO:teuthology.orchestra.run.trial178.stdout:Extra Packages for Enterprise Linux 22 MB/s | 20 MB 00:00 2026-02-22T22:36:15.063 INFO:teuthology.orchestra.run.trial078.stdout:lab-extras 2.8 MB/s | 50 kB 00:00 2026-02-22T22:36:15.898 INFO:teuthology.orchestra.run.trial178.stdout:lab-extras 2.6 MB/s | 50 kB 00:00 2026-02-22T22:36:16.274 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:16.275 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:16.275 INFO:teuthology.orchestra.run.trial078.stdout: Package Arch Version Repository Size 2026-02-22T22:36:16.275 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:16.275 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:16.275 INFO:teuthology.orchestra.run.trial078.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T22:36:16.275 INFO:teuthology.orchestra.run.trial078.stdout:Upgrading: 2026-02-22T22:36:16.275 INFO:teuthology.orchestra.run.trial078.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout:Installing dependencies: 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T22:36:16.276 INFO:teuthology.orchestra.run.trial078.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T22:36:16.277 INFO:teuthology.orchestra.run.trial078.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout:Installing weak dependencies: 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T22:36:16.278 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:16.279 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:16.279 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:16.279 INFO:teuthology.orchestra.run.trial078.stdout:Install 33 Packages 2026-02-22T22:36:16.279 INFO:teuthology.orchestra.run.trial078.stdout:Upgrade 2 Packages 2026-02-22T22:36:16.279 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:16.280 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 63 M 2026-02-22T22:36:16.280 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:17.062 INFO:teuthology.orchestra.run.trial078.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 18 MB/s | 5.5 MB 00:00 2026-02-22T22:36:17.093 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:17.094 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:17.094 INFO:teuthology.orchestra.run.trial178.stdout: Package Arch Version Repository Size 2026-02-22T22:36:17.094 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:17.094 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout:Upgrading: 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout:Installing dependencies: 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T22:36:17.095 INFO:teuthology.orchestra.run.trial178.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T22:36:17.096 INFO:teuthology.orchestra.run.trial178.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T22:36:17.097 INFO:teuthology.orchestra.run.trial178.stdout:Installing weak dependencies: 2026-02-22T22:36:17.098 INFO:teuthology.orchestra.run.trial178.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T22:36:17.098 INFO:teuthology.orchestra.run.trial178.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T22:36:17.098 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:17.098 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:17.098 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:17.098 INFO:teuthology.orchestra.run.trial178.stdout:Install 33 Packages 2026-02-22T22:36:17.098 INFO:teuthology.orchestra.run.trial178.stdout:Upgrade 2 Packages 2026-02-22T22:36:17.098 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:17.099 INFO:teuthology.orchestra.run.trial078.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 728 kB/s | 25 kB 00:00 2026-02-22T22:36:17.099 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 63 M 2026-02-22T22:36:17.099 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:17.155 INFO:teuthology.orchestra.run.trial078.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 17 MB/s | 999 kB 00:00 2026-02-22T22:36:17.206 INFO:teuthology.orchestra.run.trial078.stdout:(4/35): libradosstriper1-19.2.3-689.gf90f4eae.e 9.7 MB/s | 503 kB 00:00 2026-02-22T22:36:17.451 INFO:teuthology.orchestra.run.trial078.stdout:(5/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9.x 15 MB/s | 11 MB 00:00 2026-02-22T22:36:17.480 INFO:teuthology.orchestra.run.trial078.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 1.5 MB/s | 45 kB 00:00 2026-02-22T22:36:17.487 INFO:teuthology.orchestra.run.trial078.stdout:(7/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_64 19 MB/s | 5.4 MB 00:00 2026-02-22T22:36:17.503 INFO:teuthology.orchestra.run.trial078.stdout:(8/35): python3-ceph-common-19.2.3-689.gf90f4ea 6.4 MB/s | 142 kB 00:00 2026-02-22T22:36:17.512 INFO:teuthology.orchestra.run.trial078.stdout:(9/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 6.7 MB/s | 165 kB 00:00 2026-02-22T22:36:17.532 INFO:teuthology.orchestra.run.trial078.stdout:(10/35): python3-rados-19.2.3-689.gf90f4eae.el9 11 MB/s | 322 kB 00:00 2026-02-22T22:36:17.542 INFO:teuthology.orchestra.run.trial078.stdout:(11/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 9.7 MB/s | 303 kB 00:00 2026-02-22T22:36:17.559 INFO:teuthology.orchestra.run.trial078.stdout:(12/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 3.5 MB/s | 100 kB 00:00 2026-02-22T22:36:17.673 INFO:teuthology.orchestra.run.trial078.stdout:(13/35): ceph-common-19.2.3-689.gf90f4eae.el9.x 24 MB/s | 22 MB 00:00 2026-02-22T22:36:17.951 INFO:teuthology.orchestra.run.trial078.stdout:(14/35): mailcap-2.1.49-5.el9.noarch.rpm 81 kB/s | 33 kB 00:00 2026-02-22T22:36:18.031 INFO:teuthology.orchestra.run.trial178.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 7.5 MB/s | 5.5 MB 00:00 2026-02-22T22:36:18.043 INFO:teuthology.orchestra.run.trial178.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 2.1 MB/s | 25 kB 00:00 2026-02-22T22:36:18.104 INFO:teuthology.orchestra.run.trial178.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 16 MB/s | 999 kB 00:00 2026-02-22T22:36:18.141 INFO:teuthology.orchestra.run.trial178.stdout:(4/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9.x 13 MB/s | 11 MB 00:00 2026-02-22T22:36:18.142 INFO:teuthology.orchestra.run.trial178.stdout:(5/35): libradosstriper1-19.2.3-689.gf90f4eae.e 13 MB/s | 503 kB 00:00 2026-02-22T22:36:18.161 INFO:teuthology.orchestra.run.trial178.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 2.4 MB/s | 45 kB 00:00 2026-02-22T22:36:18.182 INFO:teuthology.orchestra.run.trial178.stdout:(7/35): python3-ceph-common-19.2.3-689.gf90f4ea 6.6 MB/s | 142 kB 00:00 2026-02-22T22:36:18.204 INFO:teuthology.orchestra.run.trial178.stdout:(8/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 7.4 MB/s | 165 kB 00:00 2026-02-22T22:36:18.228 INFO:teuthology.orchestra.run.trial078.stdout:(15/35): pciutils-3.7.0-7.el9.x86_64.rpm 168 kB/s | 93 kB 00:00 2026-02-22T22:36:18.230 INFO:teuthology.orchestra.run.trial178.stdout:(9/35): python3-rados-19.2.3-689.gf90f4eae.el9. 12 MB/s | 322 kB 00:00 2026-02-22T22:36:18.257 INFO:teuthology.orchestra.run.trial178.stdout:(10/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 11 MB/s | 303 kB 00:00 2026-02-22T22:36:18.281 INFO:teuthology.orchestra.run.trial178.stdout:(11/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 4.1 MB/s | 100 kB 00:00 2026-02-22T22:36:18.326 INFO:teuthology.orchestra.run.trial178.stdout:(12/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_6 29 MB/s | 5.4 MB 00:00 2026-02-22T22:36:18.355 INFO:teuthology.orchestra.run.trial078.stdout:(16/35): nvme-cli-2.16-1.el9.x86_64.rpm 1.4 MB/s | 1.2 MB 00:00 2026-02-22T22:36:18.530 INFO:teuthology.orchestra.run.trial178.stdout:(13/35): ceph-common-19.2.3-689.gf90f4eae.el9.x 18 MB/s | 22 MB 00:01 2026-02-22T22:36:18.629 INFO:teuthology.orchestra.run.trial078.stdout:(17/35): boost-program-options-1.75.0-13.el9.x8 153 kB/s | 104 kB 00:00 2026-02-22T22:36:18.689 INFO:teuthology.orchestra.run.trial178.stdout:(14/35): mailcap-2.1.49-5.el9.noarch.rpm 81 kB/s | 33 kB 00:00 2026-02-22T22:36:18.735 INFO:teuthology.orchestra.run.trial178.stdout:(15/35): boost-program-options-1.75.0-13.el9.x8 2.2 MB/s | 104 kB 00:00 2026-02-22T22:36:18.752 INFO:teuthology.orchestra.run.trial178.stdout:(16/35): libnbd-1.20.3-4.el9.x86_64.rpm 9.1 MB/s | 164 kB 00:00 2026-02-22T22:36:18.764 INFO:teuthology.orchestra.run.trial178.stdout:(17/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 14 MB/s | 160 kB 00:00 2026-02-22T22:36:18.773 INFO:teuthology.orchestra.run.trial178.stdout:(18/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 4.9 MB/s | 45 kB 00:00 2026-02-22T22:36:18.794 INFO:teuthology.orchestra.run.trial178.stdout:(19/35): librdkafka-1.6.1-102.el9.x86_64.rpm 31 MB/s | 662 kB 00:00 2026-02-22T22:36:18.805 INFO:teuthology.orchestra.run.trial178.stdout:(20/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 26 MB/s | 292 kB 00:00 2026-02-22T22:36:18.816 INFO:teuthology.orchestra.run.trial178.stdout:(21/35): qatlib-25.08.0-2.el9.x86_64.rpm 21 MB/s | 240 kB 00:00 2026-02-22T22:36:18.826 INFO:teuthology.orchestra.run.trial178.stdout:(22/35): qatlib-service-25.08.0-2.el9.x86_64.rp 4.0 MB/s | 37 kB 00:00 2026-02-22T22:36:18.835 INFO:teuthology.orchestra.run.trial178.stdout:(23/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 7.0 MB/s | 66 kB 00:00 2026-02-22T22:36:18.945 INFO:teuthology.orchestra.run.trial078.stdout:(18/35): libnbd-1.20.3-4.el9.x86_64.rpm 228 kB/s | 164 kB 00:00 2026-02-22T22:36:18.998 INFO:teuthology.orchestra.run.trial178.stdout:(24/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 1.8 MB/s | 308 kB 00:00 2026-02-22T22:36:19.048 INFO:teuthology.orchestra.run.trial178.stdout:(25/35): pciutils-3.7.0-7.el9.x86_64.rpm 180 kB/s | 93 kB 00:00 2026-02-22T22:36:19.073 INFO:teuthology.orchestra.run.trial078.stdout:(19/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 223 kB/s | 160 kB 00:00 2026-02-22T22:36:19.113 INFO:teuthology.orchestra.run.trial178.stdout:(26/35): libarrow-9.0.0-15.el9.x86_64.rpm 38 MB/s | 4.4 MB 00:00 2026-02-22T22:36:19.124 INFO:teuthology.orchestra.run.trial178.stdout:(27/35): nvme-cli-2.16-1.el9.x86_64.rpm 1.4 MB/s | 1.2 MB 00:00 2026-02-22T22:36:19.133 INFO:teuthology.orchestra.run.trial178.stdout:(28/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 292 kB/s | 25 kB 00:00 2026-02-22T22:36:19.134 INFO:teuthology.orchestra.run.trial178.stdout:(29/35): liboath-2.6.12-1.el9.x86_64.rpm 2.3 MB/s | 49 kB 00:00 2026-02-22T22:36:19.156 INFO:teuthology.orchestra.run.trial178.stdout:(30/35): re2-20211101-20.el9.x86_64.rpm 8.4 MB/s | 191 kB 00:00 2026-02-22T22:36:19.193 INFO:teuthology.orchestra.run.trial178.stdout:(31/35): thrift-0.15.0-4.el9.x86_64.rpm 43 MB/s | 1.6 MB 00:00 2026-02-22T22:36:19.210 INFO:teuthology.orchestra.run.trial078.stdout:(20/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 78 kB/s | 45 kB 00:00 2026-02-22T22:36:19.238 INFO:teuthology.orchestra.run.trial178.stdout:(32/35): libunwind-1.6.2-1.el9.x86_64.rpm 590 kB/s | 67 kB 00:00 2026-02-22T22:36:19.245 INFO:teuthology.orchestra.run.trial178.stdout:(33/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 7.3 MB/s | 838 kB 00:00 2026-02-22T22:36:19.337 INFO:teuthology.orchestra.run.trial178.stdout:(34/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 32 MB/s | 3.2 MB 00:00 2026-02-22T22:36:19.363 INFO:teuthology.orchestra.run.trial178.stdout:(35/35): librados2-19.2.3-689.gf90f4eae.el9.x86 20 MB/s | 3.4 MB 00:00 2026-02-22T22:36:19.368 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:19.368 INFO:teuthology.orchestra.run.trial178.stdout:Total 28 MB/s | 63 MB 00:02 2026-02-22T22:36:19.470 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:19.489 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:19.489 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:19.743 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:19.743 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:19.817 INFO:teuthology.orchestra.run.trial078.stdout:(21/35): librdkafka-1.6.1-102.el9.x86_64.rpm 759 kB/s | 662 kB 00:00 2026-02-22T22:36:19.861 INFO:teuthology.orchestra.run.trial078.stdout:(22/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 371 kB/s | 292 kB 00:00 2026-02-22T22:36:19.974 INFO:teuthology.orchestra.run.trial078.stdout:(23/35): qatlib-25.08.0-2.el9.x86_64.rpm 314 kB/s | 240 kB 00:00 2026-02-22T22:36:20.035 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:20.044 INFO:teuthology.orchestra.run.trial178.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T22:36:20.047 INFO:teuthology.orchestra.run.trial178.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T22:36:20.058 INFO:teuthology.orchestra.run.trial178.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T22:36:20.198 INFO:teuthology.orchestra.run.trial078.stdout:(24/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 1.3 MB/s | 308 kB 00:00 2026-02-22T22:36:20.210 INFO:teuthology.orchestra.run.trial178.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T22:36:20.214 INFO:teuthology.orchestra.run.trial178.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:36:20.243 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:36:20.287 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:36:20.288 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:36:20.299 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:36:20.304 INFO:teuthology.orchestra.run.trial178.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T22:36:20.312 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T22:36:20.334 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:36:20.335 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:36:20.347 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:36:20.373 INFO:teuthology.orchestra.run.trial078.stdout:(25/35): libarrow-9.0.0-15.el9.x86_64.rpm 25 MB/s | 4.4 MB 00:00 2026-02-22T22:36:20.378 INFO:teuthology.orchestra.run.trial178.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T22:36:20.382 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T22:36:20.385 INFO:teuthology.orchestra.run.trial078.stdout:(26/35): qatlib-service-25.08.0-2.el9.x86_64.rp 65 kB/s | 37 kB 00:00 2026-02-22T22:36:20.388 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T22:36:20.391 INFO:teuthology.orchestra.run.trial178.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T22:36:20.393 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T22:36:20.398 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T22:36:20.407 INFO:teuthology.orchestra.run.trial078.stdout:(27/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 728 kB/s | 25 kB 00:00 2026-02-22T22:36:20.442 INFO:teuthology.orchestra.run.trial078.stdout:(28/35): libunwind-1.6.2-1.el9.x86_64.rpm 1.9 MB/s | 67 kB 00:00 2026-02-22T22:36:20.480 INFO:teuthology.orchestra.run.trial078.stdout:(29/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 21 MB/s | 838 kB 00:00 2026-02-22T22:36:20.483 INFO:teuthology.orchestra.run.trial078.stdout:(30/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 107 kB/s | 66 kB 00:00 2026-02-22T22:36:20.515 INFO:teuthology.orchestra.run.trial078.stdout:(31/35): re2-20211101-20.el9.x86_64.rpm 5.3 MB/s | 191 kB 00:00 2026-02-22T22:36:20.517 INFO:teuthology.orchestra.run.trial078.stdout:(32/35): liboath-2.6.12-1.el9.x86_64.rpm 370 kB/s | 49 kB 00:00 2026-02-22T22:36:20.538 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:36:20.539 INFO:teuthology.orchestra.run.trial178.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:36:20.565 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:36:20.568 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T22:36:20.574 INFO:teuthology.orchestra.run.trial178.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T22:36:20.673 INFO:teuthology.orchestra.run.trial078.stdout:(33/35): librados2-19.2.3-689.gf90f4eae.el9.x86 22 MB/s | 3.4 MB 00:00 2026-02-22T22:36:20.677 INFO:teuthology.orchestra.run.trial078.stdout:(34/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 20 MB/s | 3.2 MB 00:00 2026-02-22T22:36:20.780 INFO:teuthology.orchestra.run.trial078.stdout:(35/35): thrift-0.15.0-4.el9.x86_64.rpm 5.3 MB/s | 1.6 MB 00:00 2026-02-22T22:36:20.783 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:20.783 INFO:teuthology.orchestra.run.trial078.stdout:Total 14 MB/s | 63 MB 00:04 2026-02-22T22:36:20.797 INFO:teuthology.orchestra.run.trial178.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T22:36:20.798 INFO:teuthology.orchestra.run.trial178.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:36:20.817 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:36:20.821 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T22:36:20.828 INFO:teuthology.orchestra.run.trial178.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T22:36:20.831 INFO:teuthology.orchestra.run.trial178.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T22:36:20.831 INFO:teuthology.orchestra.run.trial178.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:36:20.851 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:36:20.888 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:20.909 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:20.909 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:20.965 INFO:teuthology.orchestra.run.trial178.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T22:36:20.968 INFO:teuthology.orchestra.run.trial178.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:36:21.166 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:21.166 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:21.252 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:36:21.252 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T22:36:21.252 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:21.288 INFO:teuthology.orchestra.run.trial178.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T22:36:21.290 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T22:36:21.464 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:21.473 INFO:teuthology.orchestra.run.trial078.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T22:36:21.476 INFO:teuthology.orchestra.run.trial078.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T22:36:21.488 INFO:teuthology.orchestra.run.trial078.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T22:36:21.639 INFO:teuthology.orchestra.run.trial078.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T22:36:21.643 INFO:teuthology.orchestra.run.trial078.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:36:21.672 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:36:21.714 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:36:21.715 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:36:21.725 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:36:21.730 INFO:teuthology.orchestra.run.trial078.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T22:36:21.739 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T22:36:21.761 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:36:21.762 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:36:21.776 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:36:21.813 INFO:teuthology.orchestra.run.trial078.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T22:36:21.818 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T22:36:21.824 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T22:36:21.826 INFO:teuthology.orchestra.run.trial078.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T22:36:21.829 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T22:36:21.834 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T22:36:21.975 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:36:21.976 INFO:teuthology.orchestra.run.trial078.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:36:22.003 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:36:22.006 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T22:36:22.012 INFO:teuthology.orchestra.run.trial078.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T22:36:22.239 INFO:teuthology.orchestra.run.trial078.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T22:36:22.240 INFO:teuthology.orchestra.run.trial078.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:36:22.256 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:36:22.261 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T22:36:22.265 INFO:teuthology.orchestra.run.trial078.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T22:36:22.268 INFO:teuthology.orchestra.run.trial078.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T22:36:22.269 INFO:teuthology.orchestra.run.trial078.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:36:22.270 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:36:22.272 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:36:22.287 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:36:22.405 INFO:teuthology.orchestra.run.trial078.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T22:36:22.408 INFO:teuthology.orchestra.run.trial078.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:36:22.555 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:36:22.559 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:36:22.602 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:36:22.602 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T22:36:22.602 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T22:36:22.602 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:22.606 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:36:22.704 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:36:22.705 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T22:36:22.705 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:22.740 INFO:teuthology.orchestra.run.trial078.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T22:36:22.742 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T22:36:23.726 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:36:23.729 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:36:24.011 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:36:24.014 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:36:24.056 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:36:24.056 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T22:36:24.056 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T22:36:24.056 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:24.060 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:36:27.092 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:36:27.092 INFO:teuthology.orchestra.run.trial178.stdout:skipping the directory /sys 2026-02-22T22:36:27.093 INFO:teuthology.orchestra.run.trial178.stdout:skipping the directory /proc 2026-02-22T22:36:27.093 INFO:teuthology.orchestra.run.trial178.stdout:skipping the directory /mnt 2026-02-22T22:36:27.093 INFO:teuthology.orchestra.run.trial178.stdout:skipping the directory /var/tmp 2026-02-22T22:36:27.093 INFO:teuthology.orchestra.run.trial178.stdout:skipping the directory /home 2026-02-22T22:36:27.093 INFO:teuthology.orchestra.run.trial178.stdout:skipping the directory /root 2026-02-22T22:36:27.093 INFO:teuthology.orchestra.run.trial178.stdout:skipping the directory /tmp 2026-02-22T22:36:27.093 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:27.548 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:36:27.572 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:36:27.573 INFO:teuthology.orchestra.run.trial178.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:27.573 INFO:teuthology.orchestra.run.trial178.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T22:36:27.573 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:36:27.573 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:36:27.573 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:27.574 INFO:teuthology.orchestra.run.trial178.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:36:27.587 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:36:27.588 INFO:teuthology.orchestra.run.trial178.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:36:27.645 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:36:27.645 INFO:teuthology.orchestra.run.trial178.stdout:Creating group 'qat' with GID 992. 2026-02-22T22:36:27.645 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:27.930 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T22:36:27.930 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T22:36:27.930 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T22:36:27.930 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T22:36:27.930 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:36:27.931 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:36:27.931 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:36:27.931 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T22:36:27.931 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T22:36:27.931 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T22:36:27.935 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T22:36:27.936 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T22:36:27.991 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:36:27.992 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:27.992 INFO:teuthology.orchestra.run.trial178.stdout:Upgraded: 2026-02-22T22:36:27.992 INFO:teuthology.orchestra.run.trial178.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.992 INFO:teuthology.orchestra.run.trial178.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.992 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:27.992 INFO:teuthology.orchestra.run.trial178.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T22:36:27.992 INFO:teuthology.orchestra.run.trial178.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.992 INFO:teuthology.orchestra.run.trial178.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T22:36:27.993 INFO:teuthology.orchestra.run.trial178.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.994 INFO:teuthology.orchestra.run.trial178.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:27.995 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:28.035 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-test 2026-02-22T22:36:28.348 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:13 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:28.433 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout: Package Arch Version Repository Size 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout:Installing dependencies: 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T22:36:28.434 INFO:teuthology.orchestra.run.trial178.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T22:36:28.435 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:28.435 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:28.435 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:28.435 INFO:teuthology.orchestra.run.trial178.stdout:Install 5 Packages 2026-02-22T22:36:28.435 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:28.435 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 50 M 2026-02-22T22:36:28.435 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 212 M 2026-02-22T22:36:28.435 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:28.567 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:36:28.567 INFO:teuthology.orchestra.run.trial078.stdout:skipping the directory /sys 2026-02-22T22:36:28.567 INFO:teuthology.orchestra.run.trial078.stdout:skipping the directory /proc 2026-02-22T22:36:28.567 INFO:teuthology.orchestra.run.trial078.stdout:skipping the directory /mnt 2026-02-22T22:36:28.568 INFO:teuthology.orchestra.run.trial078.stdout:skipping the directory /var/tmp 2026-02-22T22:36:28.568 INFO:teuthology.orchestra.run.trial078.stdout:skipping the directory /home 2026-02-22T22:36:28.568 INFO:teuthology.orchestra.run.trial078.stdout:skipping the directory /root 2026-02-22T22:36:28.568 INFO:teuthology.orchestra.run.trial078.stdout:skipping the directory /tmp 2026-02-22T22:36:28.568 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:28.832 INFO:teuthology.orchestra.run.trial178.stdout:(1/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 1.7 MB/s | 163 kB 00:00 2026-02-22T22:36:28.869 INFO:teuthology.orchestra.run.trial178.stdout:(2/5): libxslt-1.1.34-12.el9.x86_64.rpm 1.7 MB/s | 233 kB 00:00 2026-02-22T22:36:28.892 INFO:teuthology.orchestra.run.trial178.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 2.7 MB/s | 64 kB 00:00 2026-02-22T22:36:28.959 INFO:teuthology.orchestra.run.trial178.stdout:(4/5): socat-1.7.4.1-8.el9.x86_64.rpm 2.3 MB/s | 303 kB 00:00 2026-02-22T22:36:29.023 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:36:29.046 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:36:29.046 INFO:teuthology.orchestra.run.trial078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:29.046 INFO:teuthology.orchestra.run.trial078.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T22:36:29.046 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:36:29.047 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:36:29.047 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:29.048 INFO:teuthology.orchestra.run.trial078.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:36:29.060 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:36:29.060 INFO:teuthology.orchestra.run.trial078.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:36:29.118 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:36:29.118 INFO:teuthology.orchestra.run.trial078.stdout:Creating group 'qat' with GID 992. 2026-02-22T22:36:29.118 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:29.404 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T22:36:29.404 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T22:36:29.404 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T22:36:29.405 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T22:36:29.406 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T22:36:29.407 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T22:36:29.466 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:36:29.466 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:29.466 INFO:teuthology.orchestra.run.trial078.stdout:Upgraded: 2026-02-22T22:36:29.466 INFO:teuthology.orchestra.run.trial078.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.466 INFO:teuthology.orchestra.run.trial078.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.466 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:29.466 INFO:teuthology.orchestra.run.trial078.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T22:36:29.466 INFO:teuthology.orchestra.run.trial078.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T22:36:29.467 INFO:teuthology.orchestra.run.trial078.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.468 INFO:teuthology.orchestra.run.trial078.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:29.469 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:29.509 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-test 2026-02-22T22:36:29.756 INFO:teuthology.orchestra.run.trial178.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 49 MB/s | 50 MB 00:01 2026-02-22T22:36:29.757 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:29.757 INFO:teuthology.orchestra.run.trial178.stdout:Total 38 MB/s | 50 MB 00:01 2026-02-22T22:36:29.783 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:29.792 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:29.792 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:29.821 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:14 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:29.903 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout: Package Arch Version Repository Size 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout:Installing dependencies: 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T22:36:29.904 INFO:teuthology.orchestra.run.trial078.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T22:36:29.905 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:29.905 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:29.905 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:29.905 INFO:teuthology.orchestra.run.trial078.stdout:Install 5 Packages 2026-02-22T22:36:29.905 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:29.905 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 50 M 2026-02-22T22:36:29.905 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 212 M 2026-02-22T22:36:29.905 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:29.977 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:29.977 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:30.054 INFO:teuthology.orchestra.run.trial078.stdout:(1/5): libxslt-1.1.34-12.el9.x86_64.rpm 3.9 MB/s | 233 kB 00:00 2026-02-22T22:36:30.082 INFO:teuthology.orchestra.run.trial078.stdout:(2/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 1.8 MB/s | 163 kB 00:00 2026-02-22T22:36:30.084 INFO:teuthology.orchestra.run.trial078.stdout:(3/5): socat-1.7.4.1-8.el9.x86_64.rpm 9.9 MB/s | 303 kB 00:00 2026-02-22T22:36:30.109 INFO:teuthology.orchestra.run.trial078.stdout:(4/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 2.3 MB/s | 64 kB 00:00 2026-02-22T22:36:30.188 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:30.194 INFO:teuthology.orchestra.run.trial178.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T22:36:30.198 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T22:36:30.206 INFO:teuthology.orchestra.run.trial178.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T22:36:30.207 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:36:31.009 INFO:teuthology.orchestra.run.trial078.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 49 MB/s | 50 MB 00:01 2026-02-22T22:36:31.010 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:31.010 INFO:teuthology.orchestra.run.trial078.stdout:Total 46 MB/s | 50 MB 00:01 2026-02-22T22:36:31.038 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:31.048 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:31.048 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:31.233 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:31.233 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:31.444 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:31.450 INFO:teuthology.orchestra.run.trial078.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T22:36:31.454 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T22:36:31.462 INFO:teuthology.orchestra.run.trial078.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T22:36:31.463 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:36:32.582 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:36:32.600 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:36:32.839 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:36:32.839 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T22:36:32.840 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T22:36:32.840 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T22:36:32.840 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T22:36:32.894 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T22:36:32.894 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:32.894 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:32.894 INFO:teuthology.orchestra.run.trial178.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:32.894 INFO:teuthology.orchestra.run.trial178.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:32.894 INFO:teuthology.orchestra.run.trial178.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T22:36:32.894 INFO:teuthology.orchestra.run.trial178.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T22:36:32.894 INFO:teuthology.orchestra.run.trial178.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T22:36:32.895 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:32.895 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:32.926 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph 2026-02-22T22:36:33.232 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:18 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:33.319 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:33.321 INFO:teuthology.orchestra.run.trial178.stdout:====================================================================================== 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout: Package Arch Version Repository Size 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout:====================================================================================== 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout:Installing dependencies: 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T22:36:33.322 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T22:36:33.323 INFO:teuthology.orchestra.run.trial178.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T22:36:33.324 INFO:teuthology.orchestra.run.trial178.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T22:36:33.325 INFO:teuthology.orchestra.run.trial178.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T22:36:33.326 INFO:teuthology.orchestra.run.trial178.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T22:36:33.327 INFO:teuthology.orchestra.run.trial178.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T22:36:33.328 INFO:teuthology.orchestra.run.trial178.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T22:36:33.329 INFO:teuthology.orchestra.run.trial178.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout:Installing weak dependencies: 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T22:36:33.330 INFO:teuthology.orchestra.run.trial178.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T22:36:33.331 INFO:teuthology.orchestra.run.trial178.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T22:36:33.331 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:33.331 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:33.331 INFO:teuthology.orchestra.run.trial178.stdout:====================================================================================== 2026-02-22T22:36:33.331 INFO:teuthology.orchestra.run.trial178.stdout:Install 99 Packages 2026-02-22T22:36:33.331 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:33.332 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 95 M 2026-02-22T22:36:33.332 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 520 M 2026-02-22T22:36:33.332 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial178.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 125 kB/s | 6.5 kB 00:00 2026-02-22T22:36:33.756 INFO:teuthology.orchestra.run.trial178.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 7.6 MB/s | 1.1 MB 00:00 2026-02-22T22:36:33.767 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:36:33.784 INFO:teuthology.orchestra.run.trial178.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 14 MB/s | 2.4 MB 00:00 2026-02-22T22:36:33.785 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:36:33.817 INFO:teuthology.orchestra.run.trial178.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 935 kB/s | 31 kB 00:00 2026-02-22T22:36:33.855 INFO:teuthology.orchestra.run.trial178.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 3.9 MB/s | 150 kB 00:00 2026-02-22T22:36:33.893 INFO:teuthology.orchestra.run.trial178.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 21 MB/s | 4.7 MB 00:00 2026-02-22T22:36:34.010 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:36:34.011 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T22:36:34.011 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T22:36:34.011 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T22:36:34.011 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T22:36:34.057 INFO:teuthology.orchestra.run.trial178.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 19 MB/s | 3.8 MB 00:00 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:34.064 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:34.097 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph 2026-02-22T22:36:34.101 INFO:teuthology.orchestra.run.trial178.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 501 kB/s | 22 kB 00:00 2026-02-22T22:36:34.152 INFO:teuthology.orchestra.run.trial178.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 4.8 MB/s | 253 kB 00:00 2026-02-22T22:36:34.201 INFO:teuthology.orchestra.run.trial178.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 1.0 MB/s | 49 kB 00:00 2026-02-22T22:36:34.221 INFO:teuthology.orchestra.run.trial178.stdout:(11/99): ceph-mgr-diskprediction-local-19.2.3-6 23 MB/s | 7.4 MB 00:00 2026-02-22T22:36:34.238 INFO:teuthology.orchestra.run.trial178.stdout:(12/99): ceph-prometheus-alerts-19.2.3-689.gf90 450 kB/s | 17 kB 00:00 2026-02-22T22:36:34.253 INFO:teuthology.orchestra.run.trial178.stdout:(13/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 9.3 MB/s | 299 kB 00:00 2026-02-22T22:36:34.285 INFO:teuthology.orchestra.run.trial178.stdout:(14/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 16 MB/s | 769 kB 00:00 2026-02-22T22:36:34.369 INFO:teuthology.orchestra.run.trial178.stdout:(15/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 28 MB/s | 17 MB 00:00 2026-02-22T22:36:34.401 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:19 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:34.487 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:34.490 INFO:teuthology.orchestra.run.trial078.stdout:====================================================================================== 2026-02-22T22:36:34.490 INFO:teuthology.orchestra.run.trial078.stdout: Package Arch Version Repository Size 2026-02-22T22:36:34.490 INFO:teuthology.orchestra.run.trial078.stdout:====================================================================================== 2026-02-22T22:36:34.490 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:34.490 INFO:teuthology.orchestra.run.trial078.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout:Installing dependencies: 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T22:36:34.491 INFO:teuthology.orchestra.run.trial078.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T22:36:34.492 INFO:teuthology.orchestra.run.trial078.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T22:36:34.493 INFO:teuthology.orchestra.run.trial078.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T22:36:34.494 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T22:36:34.495 INFO:teuthology.orchestra.run.trial078.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T22:36:34.496 INFO:teuthology.orchestra.run.trial078.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T22:36:34.497 INFO:teuthology.orchestra.run.trial078.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout:Installing weak dependencies: 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T22:36:34.498 INFO:teuthology.orchestra.run.trial078.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T22:36:34.499 INFO:teuthology.orchestra.run.trial078.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T22:36:34.499 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:34.499 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:34.499 INFO:teuthology.orchestra.run.trial078.stdout:====================================================================================== 2026-02-22T22:36:34.499 INFO:teuthology.orchestra.run.trial078.stdout:Install 99 Packages 2026-02-22T22:36:34.499 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:34.500 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 95 M 2026-02-22T22:36:34.500 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 520 M 2026-02-22T22:36:34.500 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:34.875 INFO:teuthology.orchestra.run.trial178.stdout:(16/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 69 kB/s | 40 kB 00:00 2026-02-22T22:36:35.091 INFO:teuthology.orchestra.run.trial178.stdout:(17/99): libconfig-1.7.2-9.el9.x86_64.rpm 100 kB/s | 72 kB 00:00 2026-02-22T22:36:35.103 INFO:teuthology.orchestra.run.trial178.stdout:(18/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 413 kB/s | 351 kB 00:00 2026-02-22T22:36:35.249 INFO:teuthology.orchestra.run.trial178.stdout:(19/99): libquadmath-11.5.0-14.el9.x86_64.rpm 1.1 MB/s | 184 kB 00:00 2026-02-22T22:36:35.339 INFO:teuthology.orchestra.run.trial078.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 123 kB/s | 6.5 kB 00:00 2026-02-22T22:36:35.351 INFO:teuthology.orchestra.run.trial178.stdout:(20/99): libgfortran-11.5.0-14.el9.x86_64.rpm 1.6 MB/s | 794 kB 00:00 2026-02-22T22:36:35.430 INFO:teuthology.orchestra.run.trial078.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 7.5 MB/s | 1.1 MB 00:00 2026-02-22T22:36:35.457 INFO:teuthology.orchestra.run.trial078.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 14 MB/s | 2.4 MB 00:00 2026-02-22T22:36:35.490 INFO:teuthology.orchestra.run.trial078.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 948 kB/s | 31 kB 00:00 2026-02-22T22:36:35.511 INFO:teuthology.orchestra.run.trial178.stdout:(21/99): python3-ply-3.11-14.el9.noarch.rpm 668 kB/s | 106 kB 00:00 2026-02-22T22:36:35.528 INFO:teuthology.orchestra.run.trial078.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 3.9 MB/s | 150 kB 00:00 2026-02-22T22:36:35.565 INFO:teuthology.orchestra.run.trial078.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 21 MB/s | 4.7 MB 00:00 2026-02-22T22:36:35.566 INFO:teuthology.orchestra.run.trial178.stdout:(22/99): python3-cryptography-36.0.1-5.el9.x86_ 3.9 MB/s | 1.2 MB 00:00 2026-02-22T22:36:35.670 INFO:teuthology.orchestra.run.trial178.stdout:(23/99): python3-pycparser-2.20-6.el9.noarch.rp 851 kB/s | 135 kB 00:00 2026-02-22T22:36:35.710 INFO:teuthology.orchestra.run.trial078.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 21 MB/s | 3.8 MB 00:00 2026-02-22T22:36:35.726 INFO:teuthology.orchestra.run.trial178.stdout:(24/99): python3-pyparsing-2.4.7-9.el9.noarch.r 939 kB/s | 150 kB 00:00 2026-02-22T22:36:35.754 INFO:teuthology.orchestra.run.trial078.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 507 kB/s | 22 kB 00:00 2026-02-22T22:36:35.805 INFO:teuthology.orchestra.run.trial078.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 4.9 MB/s | 253 kB 00:00 2026-02-22T22:36:35.829 INFO:teuthology.orchestra.run.trial178.stdout:(25/99): python3-requests-2.25.1-10.el9.noarch. 797 kB/s | 126 kB 00:00 2026-02-22T22:36:35.854 INFO:teuthology.orchestra.run.trial078.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 1.0 MB/s | 49 kB 00:00 2026-02-22T22:36:35.887 INFO:teuthology.orchestra.run.trial178.stdout:(26/99): python3-urllib3-1.26.5-7.el9.noarch.rp 1.3 MB/s | 218 kB 00:00 2026-02-22T22:36:35.888 INFO:teuthology.orchestra.run.trial078.stdout:(11/99): ceph-mgr-diskprediction-local-19.2.3-6 23 MB/s | 7.4 MB 00:00 2026-02-22T22:36:35.891 INFO:teuthology.orchestra.run.trial078.stdout:(12/99): ceph-prometheus-alerts-19.2.3-689.gf90 444 kB/s | 17 kB 00:00 2026-02-22T22:36:35.920 INFO:teuthology.orchestra.run.trial078.stdout:(13/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 9.4 MB/s | 299 kB 00:00 2026-02-22T22:36:35.936 INFO:teuthology.orchestra.run.trial078.stdout:(14/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 17 MB/s | 769 kB 00:00 2026-02-22T22:36:35.987 INFO:teuthology.orchestra.run.trial178.stdout:(27/99): unzip-6.0-59.el9.x86_64.rpm 1.1 MB/s | 182 kB 00:00 2026-02-22T22:36:35.991 INFO:teuthology.orchestra.run.trial078.stdout:(15/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 739 kB/s | 40 kB 00:00 2026-02-22T22:36:36.034 INFO:teuthology.orchestra.run.trial178.stdout:(28/99): flexiblas-3.0.4-9.el9.x86_64.rpm 632 kB/s | 30 kB 00:00 2026-02-22T22:36:36.042 INFO:teuthology.orchestra.run.trial078.stdout:(16/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 28 MB/s | 17 MB 00:00 2026-02-22T22:36:36.042 INFO:teuthology.orchestra.run.trial078.stdout:(17/99): libconfig-1.7.2-9.el9.x86_64.rpm 1.4 MB/s | 72 kB 00:00 2026-02-22T22:36:36.047 INFO:teuthology.orchestra.run.trial178.stdout:(29/99): zip-3.0-35.el9.x86_64.rpm 1.6 MB/s | 266 kB 00:00 2026-02-22T22:36:36.052 INFO:teuthology.orchestra.run.trial078.stdout:(18/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 2.6 MB/s | 351 kB 00:00 2026-02-22T22:36:36.078 INFO:teuthology.orchestra.run.trial078.stdout:(19/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 9.6 MB/s | 253 kB 00:00 2026-02-22T22:36:36.081 INFO:teuthology.orchestra.run.trial078.stdout:(20/99): libquadmath-11.5.0-14.el9.x86_64.rpm 4.6 MB/s | 184 kB 00:00 2026-02-22T22:36:36.088 INFO:teuthology.orchestra.run.trial178.stdout:(30/99): flexiblas-openblas-openmp-3.0.4-9.el9. 356 kB/s | 15 kB 00:00 2026-02-22T22:36:36.136 INFO:teuthology.orchestra.run.trial178.stdout:(31/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 245 kB/s | 253 kB 00:01 2026-02-22T22:36:36.153 INFO:teuthology.orchestra.run.trial178.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 3.8 MB/s | 246 kB 00:00 2026-02-22T22:36:36.162 INFO:teuthology.orchestra.run.trial178.stdout:(33/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 23 MB/s | 3.0 MB 00:00 2026-02-22T22:36:36.174 INFO:teuthology.orchestra.run.trial078.stdout:(21/99): python3-ply-3.11-14.el9.noarch.rpm 1.1 MB/s | 106 kB 00:00 2026-02-22T22:36:36.175 INFO:teuthology.orchestra.run.trial178.stdout:(34/99): openblas-0.3.29-1.el9.x86_64.rpm 1.8 MB/s | 42 kB 00:00 2026-02-22T22:36:36.181 INFO:teuthology.orchestra.run.trial078.stdout:(22/99): python3-cryptography-36.0.1-5.el9.x86_ 12 MB/s | 1.2 MB 00:00 2026-02-22T22:36:36.195 INFO:teuthology.orchestra.run.trial078.stdout:(23/99): python3-pycparser-2.20-6.el9.noarch.rp 6.1 MB/s | 135 kB 00:00 2026-02-22T22:36:36.203 INFO:teuthology.orchestra.run.trial078.stdout:(24/99): libgfortran-11.5.0-14.el9.x86_64.rpm 4.8 MB/s | 794 kB 00:00 2026-02-22T22:36:36.206 INFO:teuthology.orchestra.run.trial078.stdout:(25/99): python3-pyparsing-2.4.7-9.el9.noarch.r 5.8 MB/s | 150 kB 00:00 2026-02-22T22:36:36.216 INFO:teuthology.orchestra.run.trial178.stdout:(35/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 99 MB/s | 5.3 MB 00:00 2026-02-22T22:36:36.217 INFO:teuthology.orchestra.run.trial078.stdout:(26/99): python3-requests-2.25.1-10.el9.noarch. 5.8 MB/s | 126 kB 00:00 2026-02-22T22:36:36.220 INFO:teuthology.orchestra.run.trial178.stdout:(36/99): lua-5.4.4-4.el9.x86_64.rpm 2.2 MB/s | 188 kB 00:00 2026-02-22T22:36:36.229 INFO:teuthology.orchestra.run.trial078.stdout:(27/99): python3-urllib3-1.26.5-7.el9.noarch.rp 8.1 MB/s | 218 kB 00:00 2026-02-22T22:36:36.232 INFO:teuthology.orchestra.run.trial078.stdout:(28/99): unzip-6.0-59.el9.x86_64.rpm 6.8 MB/s | 182 kB 00:00 2026-02-22T22:36:36.238 INFO:teuthology.orchestra.run.trial078.stdout:(29/99): zip-3.0-35.el9.x86_64.rpm 12 MB/s | 266 kB 00:00 2026-02-22T22:36:36.253 INFO:teuthology.orchestra.run.trial178.stdout:(37/99): protobuf-3.14.0-17.el9.x86_64.rpm 13 MB/s | 1.0 MB 00:00 2026-02-22T22:36:36.271 INFO:teuthology.orchestra.run.trial178.stdout:(38/99): python3-babel-2.9.1-2.el9.noarch.rpm 108 MB/s | 6.0 MB 00:00 2026-02-22T22:36:36.272 INFO:teuthology.orchestra.run.trial178.stdout:(39/99): python3-devel-3.9.25-3.el9.x86_64.rpm 4.7 MB/s | 244 kB 00:00 2026-02-22T22:36:36.277 INFO:teuthology.orchestra.run.trial178.stdout:(40/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 10 MB/s | 249 kB 00:00 2026-02-22T22:36:36.295 INFO:teuthology.orchestra.run.trial178.stdout:(41/99): python3-libstoragemgmt-1.10.1-1.el9.x8 7.2 MB/s | 177 kB 00:00 2026-02-22T22:36:36.302 INFO:teuthology.orchestra.run.trial178.stdout:(42/99): python3-mako-1.1.4-6.el9.noarch.rpm 6.8 MB/s | 172 kB 00:00 2026-02-22T22:36:36.318 INFO:teuthology.orchestra.run.trial178.stdout:(43/99): python3-markupsafe-1.1.1-12.el9.x86_64 1.5 MB/s | 35 kB 00:00 2026-02-22T22:36:36.320 INFO:teuthology.orchestra.run.trial178.stdout:(44/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 26 MB/s | 1.2 MB 00:00 2026-02-22T22:36:36.338 INFO:teuthology.orchestra.run.trial078.stdout:(30/99): flexiblas-3.0.4-9.el9.x86_64.rpm 271 kB/s | 30 kB 00:00 2026-02-22T22:36:36.342 INFO:teuthology.orchestra.run.trial178.stdout:(45/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 18 MB/s | 442 kB 00:00 2026-02-22T22:36:36.343 INFO:teuthology.orchestra.run.trial178.stdout:(46/99): python3-packaging-20.9-5.el9.noarch.rp 3.3 MB/s | 77 kB 00:00 2026-02-22T22:36:36.348 INFO:teuthology.orchestra.run.trial078.stdout:(31/99): flexiblas-openblas-openmp-3.0.4-9.el9. 135 kB/s | 15 kB 00:00 2026-02-22T22:36:36.366 INFO:teuthology.orchestra.run.trial178.stdout:(47/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 6.7 MB/s | 159 kB 00:00 2026-02-22T22:36:36.367 INFO:teuthology.orchestra.run.trial178.stdout:(48/99): python3-protobuf-3.14.0-17.el9.noarch. 11 MB/s | 267 kB 00:00 2026-02-22T22:36:36.375 INFO:teuthology.orchestra.run.trial178.stdout:(49/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 84 MB/s | 6.1 MB 00:00 2026-02-22T22:36:36.389 INFO:teuthology.orchestra.run.trial178.stdout:(50/99): python3-requests-oauthlib-1.3.0-12.el9 2.3 MB/s | 54 kB 00:00 2026-02-22T22:36:36.391 INFO:teuthology.orchestra.run.trial178.stdout:(51/99): python3-pyasn1-modules-0.4.8-6.el9.noa 11 MB/s | 279 kB 00:00 2026-02-22T22:36:36.411 INFO:teuthology.orchestra.run.trial178.stdout:(52/99): python3-toml-0.10.2-6.el9.noarch.rpm 1.8 MB/s | 42 kB 00:00 2026-02-22T22:36:36.414 INFO:teuthology.orchestra.run.trial178.stdout:(53/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 7.9 MB/s | 189 kB 00:00 2026-02-22T22:36:36.434 INFO:teuthology.orchestra.run.trial178.stdout:(54/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 3.9 MB/s | 90 kB 00:00 2026-02-22T22:36:36.441 INFO:teuthology.orchestra.run.trial078.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 2.3 MB/s | 246 kB 00:00 2026-02-22T22:36:36.448 INFO:teuthology.orchestra.run.trial078.stdout:(33/99): lua-5.4.4-4.el9.x86_64.rpm 1.8 MB/s | 188 kB 00:00 2026-02-22T22:36:36.473 INFO:teuthology.orchestra.run.trial078.stdout:(34/99): openblas-0.3.29-1.el9.x86_64.rpm 1.3 MB/s | 42 kB 00:00 2026-02-22T22:36:36.482 INFO:teuthology.orchestra.run.trial178.stdout:(55/99): lua-devel-5.4.4-4.el9.x86_64.rpm 329 kB/s | 22 kB 00:00 2026-02-22T22:36:36.520 INFO:teuthology.orchestra.run.trial078.stdout:(35/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 10 MB/s | 3.0 MB 00:00 2026-02-22T22:36:36.528 INFO:teuthology.orchestra.run.trial178.stdout:(56/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 126 MB/s | 19 MB 00:00 2026-02-22T22:36:36.548 INFO:teuthology.orchestra.run.trial178.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 979 kB/s | 19 kB 00:00 2026-02-22T22:36:36.549 INFO:teuthology.orchestra.run.trial178.stdout:(58/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 8.0 MB/s | 551 kB 00:00 2026-02-22T22:36:36.560 INFO:teuthology.orchestra.run.trial178.stdout:(59/99): python3-asyncssh-2.13.2-5.el9.noarch.r 50 MB/s | 548 kB 00:00 2026-02-22T22:36:36.568 INFO:teuthology.orchestra.run.trial178.stdout:(60/99): python3-autocommand-2.2.2-8.el9.noarch 3.7 MB/s | 29 kB 00:00 2026-02-22T22:36:36.569 INFO:teuthology.orchestra.run.trial178.stdout:(61/99): luarocks-3.9.2-5.el9.noarch.rpm 7.0 MB/s | 151 kB 00:00 2026-02-22T22:36:36.576 INFO:teuthology.orchestra.run.trial178.stdout:(62/99): python3-backports-tarfile-1.2.0-1.el9. 7.3 MB/s | 60 kB 00:00 2026-02-22T22:36:36.578 INFO:teuthology.orchestra.run.trial178.stdout:(63/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 5.2 MB/s | 43 kB 00:00 2026-02-22T22:36:36.584 INFO:teuthology.orchestra.run.trial178.stdout:(64/99): python3-cachetools-4.2.4-1.el9.noarch. 4.1 MB/s | 32 kB 00:00 2026-02-22T22:36:36.585 INFO:teuthology.orchestra.run.trial178.stdout:(65/99): python3-certifi-2023.05.07-4.el9.noarc 1.8 MB/s | 14 kB 00:00 2026-02-22T22:36:36.593 INFO:teuthology.orchestra.run.trial178.stdout:(66/99): python3-cheroot-10.0.1-4.el9.noarch.rp 21 MB/s | 173 kB 00:00 2026-02-22T22:36:36.602 INFO:teuthology.orchestra.run.trial178.stdout:(67/99): python3-cherrypy-18.6.1-2.el9.noarch.r 21 MB/s | 358 kB 00:00 2026-02-22T22:36:36.603 INFO:teuthology.orchestra.run.trial178.stdout:(68/99): protobuf-compiler-3.14.0-17.el9.x86_64 5.0 MB/s | 862 kB 00:00 2026-02-22T22:36:36.605 INFO:teuthology.orchestra.run.trial178.stdout:(69/99): python3-google-auth-2.45.0-1.el9.noarc 21 MB/s | 254 kB 00:00 2026-02-22T22:36:36.614 INFO:teuthology.orchestra.run.trial178.stdout:(70/99): python3-influxdb-5.3.1-1.el9.noarch.rp 15 MB/s | 139 kB 00:00 2026-02-22T22:36:36.622 INFO:teuthology.orchestra.run.trial178.stdout:(71/99): python3-isodate-0.6.1-3.el9.noarch.rpm 6.6 MB/s | 56 kB 00:00 2026-02-22T22:36:36.627 INFO:teuthology.orchestra.run.trial178.stdout:(72/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 80 MB/s | 2.0 MB 00:00 2026-02-22T22:36:36.630 INFO:teuthology.orchestra.run.trial178.stdout:(73/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 1.4 MB/s | 11 kB 00:00 2026-02-22T22:36:36.635 INFO:teuthology.orchestra.run.trial178.stdout:(74/99): python3-jaraco-classes-3.2.1-5.el9.noa 2.3 MB/s | 18 kB 00:00 2026-02-22T22:36:36.638 INFO:teuthology.orchestra.run.trial178.stdout:(75/99): python3-jaraco-collections-3.0.0-8.el9 3.0 MB/s | 23 kB 00:00 2026-02-22T22:36:36.640 INFO:teuthology.orchestra.run.trial178.stdout:(76/99): python3-grpcio-tools-1.46.7-10.el9.x86 3.8 MB/s | 144 kB 00:00 2026-02-22T22:36:36.642 INFO:teuthology.orchestra.run.trial178.stdout:(77/99): python3-jaraco-context-6.0.1-3.el9.noa 2.6 MB/s | 20 kB 00:00 2026-02-22T22:36:36.645 INFO:teuthology.orchestra.run.trial178.stdout:(78/99): python3-jaraco-functools-3.5.0-2.el9.n 2.5 MB/s | 19 kB 00:00 2026-02-22T22:36:36.648 INFO:teuthology.orchestra.run.trial178.stdout:(79/99): python3-jaraco-text-4.0.0-2.el9.noarch 3.2 MB/s | 26 kB 00:00 2026-02-22T22:36:36.654 INFO:teuthology.orchestra.run.trial178.stdout:(80/99): python3-logutils-0.3.5-21.el9.noarch.r 5.6 MB/s | 46 kB 00:00 2026-02-22T22:36:36.660 INFO:teuthology.orchestra.run.trial178.stdout:(81/99): python3-more-itertools-8.12.0-2.el9.no 6.9 MB/s | 79 kB 00:00 2026-02-22T22:36:36.663 INFO:teuthology.orchestra.run.trial178.stdout:(82/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 9.7 MB/s | 86 kB 00:00 2026-02-22T22:36:36.663 INFO:teuthology.orchestra.run.trial078.stdout:(36/99): python3-babel-2.9.1-2.el9.noarch.rpm 42 MB/s | 6.0 MB 00:00 2026-02-22T22:36:36.669 INFO:teuthology.orchestra.run.trial178.stdout:(83/99): python3-natsort-7.1.1-5.el9.noarch.rpm 5.8 MB/s | 58 kB 00:00 2026-02-22T22:36:36.671 INFO:teuthology.orchestra.run.trial178.stdout:(84/99): python3-kubernetes-26.1.0-3.el9.noarch 36 MB/s | 1.0 MB 00:00 2026-02-22T22:36:36.673 INFO:teuthology.orchestra.run.trial178.stdout:(85/99): python3-pecan-1.4.2-3.el9.noarch.rpm 26 MB/s | 272 kB 00:00 2026-02-22T22:36:36.673 INFO:teuthology.orchestra.run.trial078.stdout:(37/99): protobuf-3.14.0-17.el9.x86_64.rpm 5.0 MB/s | 1.0 MB 00:00 2026-02-22T22:36:36.677 INFO:teuthology.orchestra.run.trial178.stdout:(86/99): python3-portend-3.1.0-2.el9.noarch.rpm 2.1 MB/s | 16 kB 00:00 2026-02-22T22:36:36.680 INFO:teuthology.orchestra.run.trial178.stdout:(87/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 10 MB/s | 90 kB 00:00 2026-02-22T22:36:36.681 INFO:teuthology.orchestra.run.trial178.stdout:(88/99): python3-repoze-lru-0.7-16.el9.noarch.r 3.9 MB/s | 31 kB 00:00 2026-02-22T22:36:36.688 INFO:teuthology.orchestra.run.trial178.stdout:(89/99): python3-rsa-4.9-2.el9.noarch.rpm 6.9 MB/s | 59 kB 00:00 2026-02-22T22:36:36.690 INFO:teuthology.orchestra.run.trial178.stdout:(90/99): python3-saml-1.16.0-1.el9.noarch.rpm 13 MB/s | 125 kB 00:00 2026-02-22T22:36:36.693 INFO:teuthology.orchestra.run.trial178.stdout:(91/99): python3-routes-2.5.1-5.el9.noarch.rpm 12 MB/s | 188 kB 00:00 2026-02-22T22:36:36.696 INFO:teuthology.orchestra.run.trial178.stdout:(92/99): python3-tempora-5.0.0-2.el9.noarch.rpm 4.4 MB/s | 36 kB 00:00 2026-02-22T22:36:36.699 INFO:teuthology.orchestra.run.trial178.stdout:(93/99): python3-typing-extensions-4.15.0-1.el9 10 MB/s | 86 kB 00:00 2026-02-22T22:36:36.705 INFO:teuthology.orchestra.run.trial178.stdout:(94/99): python3-websocket-client-1.2.3-2.el9.n 9.9 MB/s | 90 kB 00:00 2026-02-22T22:36:36.706 INFO:teuthology.orchestra.run.trial178.stdout:(95/99): python3-webob-1.8.8-2.el9.noarch.rpm 18 MB/s | 230 kB 00:00 2026-02-22T22:36:36.709 INFO:teuthology.orchestra.run.trial178.stdout:(96/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 40 MB/s | 427 kB 00:00 2026-02-22T22:36:36.711 INFO:teuthology.orchestra.run.trial078.stdout:(38/99): python3-devel-3.9.25-3.el9.x86_64.rpm 5.0 MB/s | 244 kB 00:00 2026-02-22T22:36:36.714 INFO:teuthology.orchestra.run.trial178.stdout:(97/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 5.7 MB/s | 48 kB 00:00 2026-02-22T22:36:36.715 INFO:teuthology.orchestra.run.trial178.stdout:(98/99): python3-xmltodict-0.12.0-15.el9.noarch 2.6 MB/s | 22 kB 00:00 2026-02-22T22:36:36.718 INFO:teuthology.orchestra.run.trial178.stdout:(99/99): python3-zc-lockfile-2.0-10.el9.noarch. 2.3 MB/s | 20 kB 00:00 2026-02-22T22:36:36.719 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:36.719 INFO:teuthology.orchestra.run.trial178.stdout:Total 28 MB/s | 95 MB 00:03 2026-02-22T22:36:36.722 INFO:teuthology.orchestra.run.trial078.stdout:(39/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 5.0 MB/s | 249 kB 00:00 2026-02-22T22:36:36.758 INFO:teuthology.orchestra.run.trial078.stdout:(40/99): python3-libstoragemgmt-1.10.1-1.el9.x8 3.7 MB/s | 177 kB 00:00 2026-02-22T22:36:36.827 INFO:teuthology.orchestra.run.trial078.stdout:(41/99): python3-mako-1.1.4-6.el9.noarch.rpm 2.4 MB/s | 172 kB 00:00 2026-02-22T22:36:36.860 INFO:teuthology.orchestra.run.trial078.stdout:(42/99): python3-markupsafe-1.1.1-12.el9.x86_64 1.0 MB/s | 35 kB 00:00 2026-02-22T22:36:36.963 INFO:teuthology.orchestra.run.trial078.stdout:(43/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 5.1 MB/s | 1.2 MB 00:00 2026-02-22T22:36:37.012 INFO:teuthology.orchestra.run.trial078.stdout:(44/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 40 MB/s | 6.1 MB 00:00 2026-02-22T22:36:37.042 INFO:teuthology.orchestra.run.trial078.stdout:(45/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 5.4 MB/s | 442 kB 00:00 2026-02-22T22:36:37.050 INFO:teuthology.orchestra.run.trial078.stdout:(46/99): python3-packaging-20.9-5.el9.noarch.rp 2.0 MB/s | 77 kB 00:00 2026-02-22T22:36:37.099 INFO:teuthology.orchestra.run.trial078.stdout:(47/99): python3-protobuf-3.14.0-17.el9.noarch. 4.6 MB/s | 267 kB 00:00 2026-02-22T22:36:37.102 INFO:teuthology.orchestra.run.trial078.stdout:(48/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 3.0 MB/s | 159 kB 00:00 2026-02-22T22:36:37.147 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:37.154 INFO:teuthology.orchestra.run.trial078.stdout:(49/99): python3-requests-oauthlib-1.3.0-12.el9 1.0 MB/s | 54 kB 00:00 2026-02-22T22:36:37.157 INFO:teuthology.orchestra.run.trial078.stdout:(50/99): python3-pyasn1-modules-0.4.8-6.el9.noa 4.7 MB/s | 279 kB 00:00 2026-02-22T22:36:37.185 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:37.185 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:37.193 INFO:teuthology.orchestra.run.trial078.stdout:(51/99): python3-toml-0.10.2-6.el9.noarch.rpm 1.1 MB/s | 42 kB 00:00 2026-02-22T22:36:37.238 INFO:teuthology.orchestra.run.trial078.stdout:(52/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 4.1 MB/s | 189 kB 00:00 2026-02-22T22:36:37.277 INFO:teuthology.orchestra.run.trial078.stdout:(53/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 2.3 MB/s | 90 kB 00:00 2026-02-22T22:36:37.389 INFO:teuthology.orchestra.run.trial078.stdout:(54/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 5.6 MB/s | 5.3 MB 00:00 2026-02-22T22:36:37.527 INFO:teuthology.orchestra.run.trial078.stdout:(55/99): lua-devel-5.4.4-4.el9.x86_64.rpm 89 kB/s | 22 kB 00:00 2026-02-22T22:36:37.569 INFO:teuthology.orchestra.run.trial078.stdout:(56/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 46 MB/s | 19 MB 00:00 2026-02-22T22:36:37.592 INFO:teuthology.orchestra.run.trial078.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 831 kB/s | 19 kB 00:00 2026-02-22T22:36:37.593 INFO:teuthology.orchestra.run.trial078.stdout:(58/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 8.0 MB/s | 551 kB 00:00 2026-02-22T22:36:37.602 INFO:teuthology.orchestra.run.trial078.stdout:(59/99): python3-asyncssh-2.13.2-5.el9.noarch.r 66 MB/s | 548 kB 00:00 2026-02-22T22:36:37.603 INFO:teuthology.orchestra.run.trial078.stdout:(60/99): luarocks-3.9.2-5.el9.noarch.rpm 15 MB/s | 151 kB 00:00 2026-02-22T22:36:37.603 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:37.603 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:37.606 INFO:teuthology.orchestra.run.trial078.stdout:(61/99): python3-autocommand-2.2.2-8.el9.noarch 6.9 MB/s | 29 kB 00:00 2026-02-22T22:36:37.607 INFO:teuthology.orchestra.run.trial078.stdout:(62/99): python3-backports-tarfile-1.2.0-1.el9. 15 MB/s | 60 kB 00:00 2026-02-22T22:36:37.610 INFO:teuthology.orchestra.run.trial078.stdout:(63/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 10 MB/s | 43 kB 00:00 2026-02-22T22:36:37.611 INFO:teuthology.orchestra.run.trial078.stdout:(64/99): python3-cachetools-4.2.4-1.el9.noarch. 7.5 MB/s | 32 kB 00:00 2026-02-22T22:36:37.614 INFO:teuthology.orchestra.run.trial078.stdout:(65/99): python3-certifi-2023.05.07-4.el9.noarc 3.5 MB/s | 14 kB 00:00 2026-02-22T22:36:37.618 INFO:teuthology.orchestra.run.trial078.stdout:(66/99): python3-cheroot-10.0.1-4.el9.noarch.rp 25 MB/s | 173 kB 00:00 2026-02-22T22:36:37.620 INFO:teuthology.orchestra.run.trial078.stdout:(67/99): python3-cherrypy-18.6.1-2.el9.noarch.r 65 MB/s | 358 kB 00:00 2026-02-22T22:36:37.623 INFO:teuthology.orchestra.run.trial078.stdout:(68/99): python3-google-auth-2.45.0-1.el9.noarc 54 MB/s | 254 kB 00:00 2026-02-22T22:36:37.627 INFO:teuthology.orchestra.run.trial078.stdout:(69/99): python3-grpcio-tools-1.46.7-10.el9.x86 32 MB/s | 144 kB 00:00 2026-02-22T22:36:37.631 INFO:teuthology.orchestra.run.trial078.stdout:(70/99): python3-influxdb-5.3.1-1.el9.noarch.rp 32 MB/s | 139 kB 00:00 2026-02-22T22:36:37.634 INFO:teuthology.orchestra.run.trial078.stdout:(71/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 146 MB/s | 2.0 MB 00:00 2026-02-22T22:36:37.635 INFO:teuthology.orchestra.run.trial078.stdout:(72/99): python3-isodate-0.6.1-3.el9.noarch.rpm 15 MB/s | 56 kB 00:00 2026-02-22T22:36:37.638 INFO:teuthology.orchestra.run.trial078.stdout:(73/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 2.5 MB/s | 11 kB 00:00 2026-02-22T22:36:37.639 INFO:teuthology.orchestra.run.trial078.stdout:(74/99): python3-jaraco-classes-3.2.1-5.el9.noa 4.8 MB/s | 18 kB 00:00 2026-02-22T22:36:37.643 INFO:teuthology.orchestra.run.trial078.stdout:(75/99): python3-jaraco-collections-3.0.0-8.el9 5.7 MB/s | 23 kB 00:00 2026-02-22T22:36:37.643 INFO:teuthology.orchestra.run.trial078.stdout:(76/99): python3-jaraco-context-6.0.1-3.el9.noa 5.0 MB/s | 20 kB 00:00 2026-02-22T22:36:37.647 INFO:teuthology.orchestra.run.trial078.stdout:(77/99): python3-jaraco-functools-3.5.0-2.el9.n 4.7 MB/s | 19 kB 00:00 2026-02-22T22:36:37.647 INFO:teuthology.orchestra.run.trial078.stdout:(78/99): python3-jaraco-text-4.0.0-2.el9.noarch 6.4 MB/s | 26 kB 00:00 2026-02-22T22:36:37.651 INFO:teuthology.orchestra.run.trial078.stdout:(79/99): python3-logutils-0.3.5-21.el9.noarch.r 12 MB/s | 46 kB 00:00 2026-02-22T22:36:37.656 INFO:teuthology.orchestra.run.trial078.stdout:(80/99): python3-kubernetes-26.1.0-3.el9.noarch 117 MB/s | 1.0 MB 00:00 2026-02-22T22:36:37.656 INFO:teuthology.orchestra.run.trial078.stdout:(81/99): python3-more-itertools-8.12.0-2.el9.no 15 MB/s | 79 kB 00:00 2026-02-22T22:36:37.660 INFO:teuthology.orchestra.run.trial078.stdout:(82/99): python3-natsort-7.1.1-5.el9.noarch.rpm 14 MB/s | 58 kB 00:00 2026-02-22T22:36:37.661 INFO:teuthology.orchestra.run.trial078.stdout:(83/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 17 MB/s | 86 kB 00:00 2026-02-22T22:36:37.665 INFO:teuthology.orchestra.run.trial078.stdout:(84/99): python3-portend-3.1.0-2.el9.noarch.rpm 4.1 MB/s | 16 kB 00:00 2026-02-22T22:36:37.666 INFO:teuthology.orchestra.run.trial078.stdout:(85/99): python3-pecan-1.4.2-3.el9.noarch.rpm 52 MB/s | 272 kB 00:00 2026-02-22T22:36:37.669 INFO:teuthology.orchestra.run.trial078.stdout:(86/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 20 MB/s | 90 kB 00:00 2026-02-22T22:36:37.670 INFO:teuthology.orchestra.run.trial078.stdout:(87/99): python3-repoze-lru-0.7-16.el9.noarch.r 7.4 MB/s | 31 kB 00:00 2026-02-22T22:36:37.674 INFO:teuthology.orchestra.run.trial078.stdout:(88/99): python3-rsa-4.9-2.el9.noarch.rpm 14 MB/s | 59 kB 00:00 2026-02-22T22:36:37.675 INFO:teuthology.orchestra.run.trial078.stdout:(89/99): python3-routes-2.5.1-5.el9.noarch.rpm 34 MB/s | 188 kB 00:00 2026-02-22T22:36:37.678 INFO:teuthology.orchestra.run.trial078.stdout:(90/99): python3-saml-1.16.0-1.el9.noarch.rpm 29 MB/s | 125 kB 00:00 2026-02-22T22:36:37.679 INFO:teuthology.orchestra.run.trial078.stdout:(91/99): python3-tempora-5.0.0-2.el9.noarch.rpm 8.7 MB/s | 36 kB 00:00 2026-02-22T22:36:37.682 INFO:teuthology.orchestra.run.trial078.stdout:(92/99): python3-typing-extensions-4.15.0-1.el9 22 MB/s | 86 kB 00:00 2026-02-22T22:36:37.684 INFO:teuthology.orchestra.run.trial078.stdout:(93/99): python3-webob-1.8.8-2.el9.noarch.rpm 46 MB/s | 230 kB 00:00 2026-02-22T22:36:37.686 INFO:teuthology.orchestra.run.trial078.stdout:(94/99): python3-websocket-client-1.2.3-2.el9.n 21 MB/s | 90 kB 00:00 2026-02-22T22:36:37.690 INFO:teuthology.orchestra.run.trial078.stdout:(95/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 73 MB/s | 427 kB 00:00 2026-02-22T22:36:37.690 INFO:teuthology.orchestra.run.trial078.stdout:(96/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 12 MB/s | 48 kB 00:00 2026-02-22T22:36:37.694 INFO:teuthology.orchestra.run.trial078.stdout:(97/99): python3-xmltodict-0.12.0-15.el9.noarch 5.3 MB/s | 22 kB 00:00 2026-02-22T22:36:37.695 INFO:teuthology.orchestra.run.trial078.stdout:(98/99): python3-zc-lockfile-2.0-10.el9.noarch. 4.1 MB/s | 20 kB 00:00 2026-02-22T22:36:38.026 INFO:teuthology.orchestra.run.trial078.stdout:(99/99): protobuf-compiler-3.14.0-17.el9.x86_64 1.3 MB/s | 862 kB 00:00 2026-02-22T22:36:38.028 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:38.028 INFO:teuthology.orchestra.run.trial078.stdout:Total 27 MB/s | 95 MB 00:03 2026-02-22T22:36:38.069 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:38.071 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T22:36:38.102 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T22:36:38.107 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T22:36:38.129 INFO:teuthology.orchestra.run.trial178.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T22:36:38.135 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T22:36:38.169 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T22:36:38.178 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T22:36:38.183 INFO:teuthology.orchestra.run.trial178.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T22:36:38.186 INFO:teuthology.orchestra.run.trial178.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T22:36:38.190 INFO:teuthology.orchestra.run.trial178.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T22:36:38.213 INFO:teuthology.orchestra.run.trial178.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T22:36:38.226 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T22:36:38.229 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T22:36:38.235 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T22:36:38.238 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T22:36:38.250 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T22:36:38.256 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T22:36:38.280 INFO:teuthology.orchestra.run.trial178.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T22:36:38.283 INFO:teuthology.orchestra.run.trial178.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T22:36:38.290 INFO:teuthology.orchestra.run.trial178.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T22:36:38.315 INFO:teuthology.orchestra.run.trial178.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T22:36:38.366 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T22:36:38.379 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T22:36:38.381 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T22:36:38.387 INFO:teuthology.orchestra.run.trial178.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T22:36:38.392 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T22:36:38.397 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T22:36:38.400 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T22:36:38.415 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T22:36:38.436 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T22:36:38.441 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T22:36:38.446 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T22:36:38.457 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T22:36:38.466 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T22:36:38.472 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T22:36:38.474 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:38.492 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T22:36:38.501 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T22:36:38.513 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:38.513 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:38.556 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T22:36:38.568 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T22:36:38.583 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T22:36:38.588 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T22:36:38.596 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T22:36:38.636 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T22:36:38.930 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:38.930 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:38.963 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T22:36:38.973 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T22:36:38.977 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T22:36:38.982 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T22:36:38.984 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T22:36:39.012 INFO:teuthology.orchestra.run.trial178.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T22:36:39.056 INFO:teuthology.orchestra.run.trial178.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T22:36:39.062 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T22:36:39.066 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T22:36:39.072 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T22:36:39.075 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T22:36:39.082 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T22:36:39.085 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T22:36:39.114 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T22:36:39.125 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T22:36:39.159 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T22:36:39.384 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T22:36:39.396 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:39.398 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T22:36:39.408 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T22:36:39.411 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T22:36:39.429 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T22:36:39.435 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T22:36:39.456 INFO:teuthology.orchestra.run.trial078.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T22:36:39.462 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T22:36:39.470 INFO:teuthology.orchestra.run.trial178.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T22:36:39.473 INFO:teuthology.orchestra.run.trial178.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T22:36:39.493 INFO:teuthology.orchestra.run.trial178.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T22:36:39.495 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T22:36:39.504 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T22:36:39.509 INFO:teuthology.orchestra.run.trial078.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T22:36:39.511 INFO:teuthology.orchestra.run.trial078.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T22:36:39.516 INFO:teuthology.orchestra.run.trial078.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T22:36:39.539 INFO:teuthology.orchestra.run.trial078.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T22:36:39.551 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T22:36:39.554 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T22:36:39.560 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T22:36:39.562 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T22:36:39.573 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T22:36:39.579 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T22:36:39.603 INFO:teuthology.orchestra.run.trial078.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T22:36:39.607 INFO:teuthology.orchestra.run.trial078.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T22:36:39.614 INFO:teuthology.orchestra.run.trial078.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T22:36:39.638 INFO:teuthology.orchestra.run.trial078.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T22:36:39.689 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T22:36:39.701 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T22:36:39.704 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T22:36:39.708 INFO:teuthology.orchestra.run.trial078.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T22:36:39.714 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T22:36:39.718 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T22:36:39.721 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T22:36:39.735 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T22:36:39.756 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T22:36:39.761 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T22:36:39.766 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T22:36:39.777 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T22:36:39.786 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T22:36:39.792 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T22:36:39.812 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T22:36:39.821 INFO:teuthology.orchestra.run.trial178.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T22:36:39.821 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T22:36:39.875 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T22:36:39.887 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T22:36:39.894 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T22:36:39.903 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T22:36:39.907 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T22:36:39.915 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T22:36:39.955 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T22:36:40.281 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T22:36:40.291 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T22:36:40.295 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T22:36:40.301 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T22:36:40.303 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T22:36:40.331 INFO:teuthology.orchestra.run.trial078.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T22:36:40.374 INFO:teuthology.orchestra.run.trial078.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T22:36:40.381 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T22:36:40.384 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T22:36:40.390 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T22:36:40.393 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T22:36:40.400 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T22:36:40.404 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T22:36:40.432 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T22:36:40.441 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T22:36:40.476 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T22:36:40.569 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T22:36:40.585 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T22:36:40.595 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T22:36:40.602 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T22:36:40.616 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T22:36:40.631 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T22:36:40.701 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T22:36:40.707 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T22:36:40.716 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T22:36:40.725 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T22:36:40.729 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T22:36:40.739 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T22:36:40.769 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T22:36:40.787 INFO:teuthology.orchestra.run.trial078.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T22:36:40.789 INFO:teuthology.orchestra.run.trial078.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T22:36:40.809 INFO:teuthology.orchestra.run.trial078.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T22:36:40.820 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T22:36:40.826 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T22:36:40.832 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T22:36:40.833 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T22:36:40.852 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:36:40.852 INFO:teuthology.orchestra.run.trial178.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T22:36:40.852 INFO:teuthology.orchestra.run.trial178.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T22:36:40.852 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:40.859 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:36:40.886 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:36:40.886 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T22:36:40.887 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:40.921 INFO:teuthology.orchestra.run.trial178.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T22:36:40.985 INFO:teuthology.orchestra.run.trial178.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T22:36:40.988 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:36:41.002 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:36:41.002 INFO:teuthology.orchestra.run.trial178.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:41.002 INFO:teuthology.orchestra.run.trial178.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T22:36:41.002 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:41.136 INFO:teuthology.orchestra.run.trial078.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T22:36:41.209 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T22:36:41.699 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:36:41.726 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:36:41.726 INFO:teuthology.orchestra.run.trial178.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:41.726 INFO:teuthology.orchestra.run.trial178.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T22:36:41.726 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:36:41.726 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:36:41.727 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:41.784 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:36:41.786 INFO:teuthology.orchestra.run.trial178.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:36:41.791 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T22:36:41.810 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T22:36:41.812 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:36:41.882 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T22:36:41.899 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T22:36:41.909 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T22:36:41.916 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T22:36:41.929 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T22:36:41.948 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T22:36:42.023 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T22:36:42.032 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T22:36:42.055 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T22:36:42.084 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T22:36:42.135 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T22:36:42.141 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T22:36:42.144 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T22:36:42.146 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T22:36:42.165 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:36:42.165 INFO:teuthology.orchestra.run.trial078.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T22:36:42.165 INFO:teuthology.orchestra.run.trial078.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T22:36:42.165 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:42.173 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:36:42.200 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:36:42.200 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T22:36:42.200 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:42.235 INFO:teuthology.orchestra.run.trial078.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T22:36:42.299 INFO:teuthology.orchestra.run.trial078.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T22:36:42.301 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:36:42.305 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:36:42.308 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:36:42.316 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:36:42.316 INFO:teuthology.orchestra.run.trial078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:42.316 INFO:teuthology.orchestra.run.trial078.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T22:36:42.316 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:42.760 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:36:42.761 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:36:42.775 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:36:42.775 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:36:42.832 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:36:42.882 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T22:36:42.883 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:36:42.904 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:36:42.905 INFO:teuthology.orchestra.run.trial178.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:42.905 INFO:teuthology.orchestra.run.trial178.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T22:36:42.905 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:36:42.905 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:36:42.905 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:42.916 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:36:43.051 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:36:43.077 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:36:43.077 INFO:teuthology.orchestra.run.trial078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:43.077 INFO:teuthology.orchestra.run.trial078.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T22:36:43.078 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:36:43.078 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:36:43.078 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:43.128 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:36:43.130 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:36:43.137 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:36:43.139 INFO:teuthology.orchestra.run.trial078.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:36:43.144 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T22:36:43.151 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:36:43.152 INFO:teuthology.orchestra.run.trial178.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:43.152 INFO:teuthology.orchestra.run.trial178.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T22:36:43.152 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:36:43.152 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:36:43.152 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:43.164 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T22:36:43.165 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:36:43.253 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:36:43.273 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:36:43.274 INFO:teuthology.orchestra.run.trial178.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:43.274 INFO:teuthology.orchestra.run.trial178.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T22:36:43.274 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:36:43.274 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:36:43.274 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:43.292 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:36:43.662 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:36:43.665 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:36:43.918 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T22:36:43.919 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T22:36:43.920 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T22:36:43.920 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T22:36:43.922 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T22:36:43.923 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T22:36:43.924 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T22:36:43.989 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T22:36:43.989 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:43.989 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:43.989 INFO:teuthology.orchestra.run.trial178.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T22:36:43.989 INFO:teuthology.orchestra.run.trial178.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:43.989 INFO:teuthology.orchestra.run.trial178.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.989 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T22:36:43.990 INFO:teuthology.orchestra.run.trial178.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T22:36:43.991 INFO:teuthology.orchestra.run.trial178.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T22:36:43.992 INFO:teuthology.orchestra.run.trial178.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T22:36:43.993 INFO:teuthology.orchestra.run.trial178.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T22:36:43.993 INFO:teuthology.orchestra.run.trial178.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T22:36:43.993 INFO:teuthology.orchestra.run.trial178.stdout: zip-3.0-35.el9.x86_64 2026-02-22T22:36:43.993 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:43.993 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:44.044 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-base 2026-02-22T22:36:44.137 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:36:44.138 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:36:44.150 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:36:44.151 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:36:44.207 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:36:44.256 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T22:36:44.257 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:36:44.280 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:36:44.280 INFO:teuthology.orchestra.run.trial078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:44.280 INFO:teuthology.orchestra.run.trial078.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T22:36:44.280 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:36:44.280 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:36:44.280 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:44.292 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:36:44.352 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:29 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:44.416 INFO:teuthology.orchestra.run.trial178.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:44.434 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:44.435 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:44.435 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:44.456 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install cephadm 2026-02-22T22:36:44.518 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:36:44.520 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:36:44.542 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:36:44.542 INFO:teuthology.orchestra.run.trial078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:44.542 INFO:teuthology.orchestra.run.trial078.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T22:36:44.542 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:36:44.542 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:36:44.542 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:44.652 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:36:44.673 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:36:44.673 INFO:teuthology.orchestra.run.trial078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:44.673 INFO:teuthology.orchestra.run.trial078.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T22:36:44.673 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:36:44.673 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:36:44.674 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:44.691 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:36:44.763 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:29 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:44.827 INFO:teuthology.orchestra.run.trial178.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:44.845 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:44.846 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:44.846 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:44.867 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T22:36:45.180 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:30 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:45.264 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:45.264 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:45.264 INFO:teuthology.orchestra.run.trial178.stdout: Package Arch Version Repo Size 2026-02-22T22:36:45.264 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout:Install 1 Package 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 145 k 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 431 k 2026-02-22T22:36:45.265 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:45.352 INFO:teuthology.orchestra.run.trial178.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.6 MB/s | 145 kB 00:00 2026-02-22T22:36:45.352 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:45.352 INFO:teuthology.orchestra.run.trial178.stdout:Total 1.6 MB/s | 145 kB 00:00 2026-02-22T22:36:45.352 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:45.355 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:45.355 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:45.387 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:36:45.387 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T22:36:45.387 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T22:36:45.387 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T22:36:45.387 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T22:36:45.387 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T22:36:45.388 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T22:36:45.389 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T22:36:45.390 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T22:36:45.391 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T22:36:45.392 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T22:36:45.393 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T22:36:45.394 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T22:36:45.395 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T22:36:45.395 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T22:36:45.395 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T22:36:45.395 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T22:36:45.395 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T22:36:45.395 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T22:36:45.395 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T22:36:45.396 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:45.396 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:45.442 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:45.444 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:36:45.451 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T22:36:45.451 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:45.451 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:45.451 INFO:teuthology.orchestra.run.trial078.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T22:36:45.451 INFO:teuthology.orchestra.run.trial078.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:45.451 INFO:teuthology.orchestra.run.trial078.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.451 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:45.451 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T22:36:45.452 INFO:teuthology.orchestra.run.trial078.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T22:36:45.453 INFO:teuthology.orchestra.run.trial078.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T22:36:45.454 INFO:teuthology.orchestra.run.trial078.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T22:36:45.455 INFO:teuthology.orchestra.run.trial078.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T22:36:45.456 INFO:teuthology.orchestra.run.trial078.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T22:36:45.457 INFO:teuthology.orchestra.run.trial078.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T22:36:45.458 INFO:teuthology.orchestra.run.trial078.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: zip-3.0-35.el9.x86_64 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:45.459 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:45.468 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:36:45.468 INFO:teuthology.orchestra.run.trial178.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:45.468 INFO:teuthology.orchestra.run.trial178.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T22:36:45.468 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:45.506 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-base 2026-02-22T22:36:45.775 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:36:45.775 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:45.775 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:45.775 INFO:teuthology.orchestra.run.trial178.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:45.775 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:45.776 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:45.802 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-mgr 2026-02-22T22:36:45.823 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:30 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:45.888 INFO:teuthology.orchestra.run.trial078.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:45.906 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:45.907 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:45.907 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:45.928 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install cephadm 2026-02-22T22:36:46.112 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:31 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:46.176 INFO:teuthology.orchestra.run.trial178.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:46.194 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:46.195 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:46.195 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:46.216 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T22:36:46.238 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:31 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:46.303 INFO:teuthology.orchestra.run.trial078.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:46.321 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:46.322 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:46.322 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:46.345 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T22:36:46.534 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:31 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:46.599 INFO:teuthology.orchestra.run.trial178.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:46.617 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:46.618 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:46.618 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:46.640 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T22:36:46.657 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:31 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:46.739 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout: Package Arch Version Repo Size 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:Install 1 Package 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 145 k 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 431 k 2026-02-22T22:36:46.740 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:46.830 INFO:teuthology.orchestra.run.trial078.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.6 MB/s | 145 kB 00:00 2026-02-22T22:36:46.830 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:46.830 INFO:teuthology.orchestra.run.trial078.stdout:Total 1.6 MB/s | 145 kB 00:00 2026-02-22T22:36:46.830 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:46.833 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:46.833 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:46.867 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:46.867 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:46.923 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:46.924 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:36:46.946 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:36:46.946 INFO:teuthology.orchestra.run.trial078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:46.946 INFO:teuthology.orchestra.run.trial078.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T22:36:46.946 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:46.947 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:31 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:47.012 INFO:teuthology.orchestra.run.trial178.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:47.030 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:47.031 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:47.031 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:47.052 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-mgr-rook 2026-02-22T22:36:47.231 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:36:47.231 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:47.232 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:47.232 INFO:teuthology.orchestra.run.trial078.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:47.232 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:47.232 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:47.260 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-mgr 2026-02-22T22:36:47.365 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:32 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:47.429 INFO:teuthology.orchestra.run.trial178.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:47.448 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:47.448 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:47.448 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:47.470 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T22:36:47.573 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:32 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:47.637 INFO:teuthology.orchestra.run.trial078.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:47.655 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:47.656 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:47.656 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:47.678 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T22:36:47.784 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:32 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:47.849 INFO:teuthology.orchestra.run.trial178.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:47.867 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:47.868 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:47.868 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:47.889 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-fuse 2026-02-22T22:36:47.986 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:32 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:48.052 INFO:teuthology.orchestra.run.trial078.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:48.070 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:48.071 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:48.071 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:48.092 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T22:36:48.197 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:48.279 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout: Package Architecture Version Repository Size 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:Install 1 Package 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 1.2 M 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 3.6 M 2026-02-22T22:36:48.280 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:48.404 INFO:teuthology.orchestra.run.trial178.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 9.3 MB/s | 1.2 MB 00:00 2026-02-22T22:36:48.405 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:48.405 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:48.405 INFO:teuthology.orchestra.run.trial178.stdout:Total 9.2 MB/s | 1.2 MB 00:00 2026-02-22T22:36:48.405 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:48.409 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:48.409 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:48.446 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:48.446 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:48.470 INFO:teuthology.orchestra.run.trial078.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:48.488 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:48.489 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:48.489 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:48.510 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-mgr-rook 2026-02-22T22:36:48.548 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:48.565 INFO:teuthology.orchestra.run.trial178.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:48.813 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:48.823 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:33 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:48.871 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:48.871 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:48.871 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:48.871 INFO:teuthology.orchestra.run.trial178.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:48.871 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:48.872 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:48.888 INFO:teuthology.orchestra.run.trial078.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:48.899 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install ceph-volume 2026-02-22T22:36:48.907 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:48.908 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:48.908 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:48.929 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T22:36:49.214 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:49.234 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:49.280 INFO:teuthology.orchestra.run.trial178.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:49.298 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:49.298 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:49.298 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:49.299 INFO:teuthology.orchestra.run.trial078.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:49.317 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:49.317 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:49.317 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:49.320 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install librados-devel 2026-02-22T22:36:49.339 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-fuse 2026-02-22T22:36:49.634 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:49.653 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:34 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:49.718 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:49.718 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:49.718 INFO:teuthology.orchestra.run.trial178.stdout: Package Arch Version Repo Size 2026-02-22T22:36:49.718 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:49.718 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:49.718 INFO:teuthology.orchestra.run.trial178.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T22:36:49.719 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:49.719 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:49.719 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:49.719 INFO:teuthology.orchestra.run.trial178.stdout:Install 1 Package 2026-02-22T22:36:49.719 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:49.719 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 126 k 2026-02-22T22:36:49.719 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 456 k 2026-02-22T22:36:49.719 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:49.736 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:49.737 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:49.737 INFO:teuthology.orchestra.run.trial078.stdout: Package Architecture Version Repository Size 2026-02-22T22:36:49.737 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:49.737 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:49.737 INFO:teuthology.orchestra.run.trial078.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T22:36:49.738 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:49.738 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:49.738 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:49.738 INFO:teuthology.orchestra.run.trial078.stdout:Install 1 Package 2026-02-22T22:36:49.738 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:49.738 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 1.2 M 2026-02-22T22:36:49.738 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 3.6 M 2026-02-22T22:36:49.738 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:49.806 INFO:teuthology.orchestra.run.trial178.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-22T22:36:49.806 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:49.807 INFO:teuthology.orchestra.run.trial178.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-22T22:36:49.807 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:49.809 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:49.810 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:49.843 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:49.843 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:49.859 INFO:teuthology.orchestra.run.trial078.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 9.4 MB/s | 1.2 MB 00:00 2026-02-22T22:36:49.859 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:49.859 INFO:teuthology.orchestra.run.trial078.stdout:Total 9.4 MB/s | 1.2 MB 00:00 2026-02-22T22:36:49.860 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:49.863 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:49.863 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:49.900 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:49.900 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:49.900 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:49.917 INFO:teuthology.orchestra.run.trial178.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:49.996 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:50.004 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:50.021 INFO:teuthology.orchestra.run.trial078.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:50.054 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:50.054 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:50.054 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:50.054 INFO:teuthology.orchestra.run.trial178.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:50.054 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:50.054 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:50.086 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install libcephfs2 2026-02-22T22:36:50.230 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:50.289 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:50.290 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:50.290 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:50.290 INFO:teuthology.orchestra.run.trial078.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:50.290 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:50.290 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:50.319 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install ceph-volume 2026-02-22T22:36:50.393 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:50.458 INFO:teuthology.orchestra.run.trial178.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:50.477 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:50.477 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:50.478 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:50.499 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install libcephfs-devel 2026-02-22T22:36:50.639 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:50.704 INFO:teuthology.orchestra.run.trial078.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:36:50.722 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:50.723 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:50.723 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:50.745 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install librados-devel 2026-02-22T22:36:50.810 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:35 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:50.894 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:50.894 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout: Package Arch Version Repo Size 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout:Install 1 Package 2026-02-22T22:36:50.895 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:50.896 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 34 k 2026-02-22T22:36:50.896 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 153 k 2026-02-22T22:36:50.896 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:50.958 INFO:teuthology.orchestra.run.trial178.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 525 kB/s | 34 kB 00:00 2026-02-22T22:36:50.958 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:50.958 INFO:teuthology.orchestra.run.trial178.stdout:Total 519 kB/s | 34 kB 00:00 2026-02-22T22:36:50.958 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:50.960 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:50.960 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:50.963 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:50.963 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:50.984 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:51.000 INFO:teuthology.orchestra.run.trial178.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:51.059 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:51.063 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:51.118 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:51.118 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:51.118 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:51.118 INFO:teuthology.orchestra.run.trial178.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:51.118 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:51.118 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:51.148 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:51.148 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:51.148 INFO:teuthology.orchestra.run.trial078.stdout: Package Arch Version Repo Size 2026-02-22T22:36:51.148 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:51.148 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:51.148 INFO:teuthology.orchestra.run.trial078.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T22:36:51.148 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:51.149 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:51.149 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:51.149 INFO:teuthology.orchestra.run.trial078.stdout:Install 1 Package 2026-02-22T22:36:51.149 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:51.149 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 126 k 2026-02-22T22:36:51.149 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 456 k 2026-02-22T22:36:51.149 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:51.151 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install librados2 2026-02-22T22:36:51.236 INFO:teuthology.orchestra.run.trial078.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-22T22:36:51.237 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:51.237 INFO:teuthology.orchestra.run.trial078.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-22T22:36:51.237 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:51.239 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:51.239 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:51.272 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:51.272 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:51.327 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:51.343 INFO:teuthology.orchestra.run.trial078.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:51.427 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:51.464 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:51.486 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:51.486 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:51.486 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:51.486 INFO:teuthology.orchestra.run.trial078.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:51.486 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:51.486 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:51.516 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install libcephfs2 2026-02-22T22:36:51.530 INFO:teuthology.orchestra.run.trial178.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:51.549 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:51.549 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:51.549 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:51.572 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install librbd1 2026-02-22T22:36:51.827 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:51.877 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:36 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:51.894 INFO:teuthology.orchestra.run.trial078.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:51.912 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:51.913 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:51.913 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:51.934 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install libcephfs-devel 2026-02-22T22:36:51.941 INFO:teuthology.orchestra.run.trial178.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:51.960 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:51.961 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:51.961 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:51.982 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install python3-rados 2026-02-22T22:36:52.256 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:52.296 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:52.341 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:52.341 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:52.341 INFO:teuthology.orchestra.run.trial078.stdout: Package Arch Version Repo Size 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout:Install 1 Package 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 34 k 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 153 k 2026-02-22T22:36:52.342 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:52.361 INFO:teuthology.orchestra.run.trial178.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:52.380 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:52.381 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:52.381 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:52.403 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install python3-rgw 2026-02-22T22:36:52.405 INFO:teuthology.orchestra.run.trial078.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 521 kB/s | 34 kB 00:00 2026-02-22T22:36:52.406 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:52.407 INFO:teuthology.orchestra.run.trial078.stdout:Total 511 kB/s | 34 kB 00:00 2026-02-22T22:36:52.407 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:52.408 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:52.409 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:52.412 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:52.412 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:52.433 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:52.449 INFO:teuthology.orchestra.run.trial078.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:52.506 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:52.564 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:52.564 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:52.564 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:52.564 INFO:teuthology.orchestra.run.trial078.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:52.564 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:52.564 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:52.590 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install librados2 2026-02-22T22:36:52.713 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:52.778 INFO:teuthology.orchestra.run.trial178.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:52.796 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:52.797 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:52.797 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:52.819 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install python3-cephfs 2026-02-22T22:36:52.904 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:37 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:52.968 INFO:teuthology.orchestra.run.trial078.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:52.986 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:52.987 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:52.987 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:53.008 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install librbd1 2026-02-22T22:36:53.124 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:53.189 INFO:teuthology.orchestra.run.trial178.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:53.208 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:53.208 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:53.208 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:53.231 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install python3-rbd 2026-02-22T22:36:53.317 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:53.382 INFO:teuthology.orchestra.run.trial078.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:53.400 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:53.401 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:53.401 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:53.421 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install python3-rados 2026-02-22T22:36:53.545 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:53.610 INFO:teuthology.orchestra.run.trial178.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:53.628 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:53.628 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:53.629 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:53.650 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install rbd-fuse 2026-02-22T22:36:53.737 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:53.802 INFO:teuthology.orchestra.run.trial078.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:53.821 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:53.821 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:53.821 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:53.844 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install python3-rgw 2026-02-22T22:36:53.964 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:38 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:54.049 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:54.049 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:54.049 INFO:teuthology.orchestra.run.trial178.stdout: Package Architecture Version Repository Size 2026-02-22T22:36:54.049 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:54.049 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:54.049 INFO:teuthology.orchestra.run.trial178.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T22:36:54.049 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:54.050 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:54.050 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:54.050 INFO:teuthology.orchestra.run.trial178.stdout:Install 1 Package 2026-02-22T22:36:54.050 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:54.050 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 85 k 2026-02-22T22:36:54.050 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 227 k 2026-02-22T22:36:54.050 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:54.125 INFO:teuthology.orchestra.run.trial178.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.1 MB/s | 85 kB 00:00 2026-02-22T22:36:54.125 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:54.126 INFO:teuthology.orchestra.run.trial178.stdout:Total 1.1 MB/s | 85 kB 00:00 2026-02-22T22:36:54.126 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:54.129 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:54.129 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:54.157 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:54.162 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:54.162 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:54.214 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:54.222 INFO:teuthology.orchestra.run.trial078.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:54.231 INFO:teuthology.orchestra.run.trial178.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:54.241 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:54.241 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:54.241 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:54.263 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install python3-cephfs 2026-02-22T22:36:54.311 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:54.370 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:54.370 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:54.370 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:54.370 INFO:teuthology.orchestra.run.trial178.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:54.370 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:54.371 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:54.397 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install rbd-mirror 2026-02-22T22:36:54.577 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:54.642 INFO:teuthology.orchestra.run.trial078.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:54.660 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:54.661 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:54.661 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:54.682 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install python3-rbd 2026-02-22T22:36:54.708 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:54.792 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:54.792 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:54.792 INFO:teuthology.orchestra.run.trial178.stdout: Package Arch Version Repo Size 2026-02-22T22:36:54.792 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:54.792 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:54.792 INFO:teuthology.orchestra.run.trial178.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T22:36:54.793 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:54.793 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:54.793 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:54.793 INFO:teuthology.orchestra.run.trial178.stdout:Install 1 Package 2026-02-22T22:36:54.793 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:54.793 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 3.1 M 2026-02-22T22:36:54.793 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 13 M 2026-02-22T22:36:54.793 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:54.944 INFO:teuthology.orchestra.run.trial178.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 21 MB/s | 3.1 MB 00:00 2026-02-22T22:36:54.945 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:54.945 INFO:teuthology.orchestra.run.trial178.stdout:Total 20 MB/s | 3.1 MB 00:00 2026-02-22T22:36:54.945 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:54.949 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:54.949 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:54.990 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:54.990 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:54.997 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:55.061 INFO:teuthology.orchestra.run.trial078.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:55.079 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:55.080 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:55.080 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:55.100 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install rbd-fuse 2026-02-22T22:36:55.189 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:55.190 INFO:teuthology.orchestra.run.trial178.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:55.214 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:55.215 INFO:teuthology.orchestra.run.trial178.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:55.215 INFO:teuthology.orchestra.run.trial178.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T22:36:55.215 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:36:55.215 INFO:teuthology.orchestra.run.trial178.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:36:55.215 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:55.412 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:55.496 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:55.496 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:55.496 INFO:teuthology.orchestra.run.trial078.stdout: Package Architecture Version Repository Size 2026-02-22T22:36:55.497 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:55.497 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:55.497 INFO:teuthology.orchestra.run.trial078.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T22:36:55.497 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:55.497 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:55.497 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:55.498 INFO:teuthology.orchestra.run.trial078.stdout:Install 1 Package 2026-02-22T22:36:55.498 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:55.498 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 85 k 2026-02-22T22:36:55.498 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 227 k 2026-02-22T22:36:55.498 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:55.505 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:55.505 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:55.505 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:55.505 INFO:teuthology.orchestra.run.trial178.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:55.505 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:55.505 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:55.533 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install rbd-nbd 2026-02-22T22:36:55.572 INFO:teuthology.orchestra.run.trial078.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.1 MB/s | 85 kB 00:00 2026-02-22T22:36:55.572 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:55.572 INFO:teuthology.orchestra.run.trial078.stdout:Total 1.1 MB/s | 85 kB 00:00 2026-02-22T22:36:55.572 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:55.575 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:55.575 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:55.607 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:55.607 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:55.658 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:55.675 INFO:teuthology.orchestra.run.trial078.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:55.757 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:55.816 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:55.816 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:55.816 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:55.816 INFO:teuthology.orchestra.run.trial078.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:55.816 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:55.816 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:55.844 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install rbd-mirror 2026-02-22T22:36:55.845 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:55.928 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:55.929 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:55.930 INFO:teuthology.orchestra.run.trial178.stdout: Package Architecture Version Repository Size 2026-02-22T22:36:55.930 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:55.930 INFO:teuthology.orchestra.run.trial178.stdout:Installing: 2026-02-22T22:36:55.930 INFO:teuthology.orchestra.run.trial178.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T22:36:55.930 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:55.930 INFO:teuthology.orchestra.run.trial178.stdout:Transaction Summary 2026-02-22T22:36:55.930 INFO:teuthology.orchestra.run.trial178.stdout:================================================================================ 2026-02-22T22:36:55.931 INFO:teuthology.orchestra.run.trial178.stdout:Install 1 Package 2026-02-22T22:36:55.931 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:55.931 INFO:teuthology.orchestra.run.trial178.stdout:Total download size: 170 k 2026-02-22T22:36:55.931 INFO:teuthology.orchestra.run.trial178.stdout:Installed size: 490 k 2026-02-22T22:36:55.931 INFO:teuthology.orchestra.run.trial178.stdout:Downloading Packages: 2026-02-22T22:36:56.017 INFO:teuthology.orchestra.run.trial178.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.9 MB/s | 170 kB 00:00 2026-02-22T22:36:56.018 INFO:teuthology.orchestra.run.trial178.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:56.018 INFO:teuthology.orchestra.run.trial178.stdout:Total 1.9 MB/s | 170 kB 00:00 2026-02-22T22:36:56.019 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction check 2026-02-22T22:36:56.022 INFO:teuthology.orchestra.run.trial178.stdout:Transaction check succeeded. 2026-02-22T22:36:56.022 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction test 2026-02-22T22:36:56.053 INFO:teuthology.orchestra.run.trial178.stdout:Transaction test succeeded. 2026-02-22T22:36:56.053 INFO:teuthology.orchestra.run.trial178.stdout:Running transaction 2026-02-22T22:36:56.109 INFO:teuthology.orchestra.run.trial178.stdout: Preparing : 1/1 2026-02-22T22:36:56.125 INFO:teuthology.orchestra.run.trial178.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:56.165 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:56.206 INFO:teuthology.orchestra.run.trial178.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:56.250 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:56.250 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:56.250 INFO:teuthology.orchestra.run.trial078.stdout: Package Arch Version Repo Size 2026-02-22T22:36:56.250 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:56.250 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:56.250 INFO:teuthology.orchestra.run.trial078.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T22:36:56.251 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:56.251 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:56.251 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:56.251 INFO:teuthology.orchestra.run.trial078.stdout:Install 1 Package 2026-02-22T22:36:56.251 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:56.251 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 3.1 M 2026-02-22T22:36:56.251 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 13 M 2026-02-22T22:36:56.251 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:56.263 INFO:teuthology.orchestra.run.trial178.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:56.263 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:56.264 INFO:teuthology.orchestra.run.trial178.stdout:Installed: 2026-02-22T22:36:56.264 INFO:teuthology.orchestra.run.trial178.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:56.264 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:36:56.264 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:56.293 DEBUG:teuthology.orchestra.run.trial178:> sudo yum -y install rbd-mirror 2026-02-22T22:36:56.403 INFO:teuthology.orchestra.run.trial078.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 20 MB/s | 3.1 MB 00:00 2026-02-22T22:36:56.404 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:56.404 INFO:teuthology.orchestra.run.trial078.stdout:Total 20 MB/s | 3.1 MB 00:00 2026-02-22T22:36:56.404 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:56.408 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:56.408 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:56.449 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:56.449 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:56.605 INFO:teuthology.orchestra.run.trial178.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:56.641 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:56.643 INFO:teuthology.orchestra.run.trial078.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:56.667 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:56.667 INFO:teuthology.orchestra.run.trial078.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:36:56.667 INFO:teuthology.orchestra.run.trial078.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T22:36:56.667 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:36:56.667 INFO:teuthology.orchestra.run.trial078.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:36:56.667 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:56.670 INFO:teuthology.orchestra.run.trial178.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:56.688 INFO:teuthology.orchestra.run.trial178.stdout:Dependencies resolved. 2026-02-22T22:36:56.689 INFO:teuthology.orchestra.run.trial178.stdout:Nothing to do. 2026-02-22T22:36:56.689 INFO:teuthology.orchestra.run.trial178.stdout:Complete! 2026-02-22T22:36:56.714 DEBUG:teuthology.parallel:result is None 2026-02-22T22:36:56.962 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:56.962 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:56.962 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:56.962 INFO:teuthology.orchestra.run.trial078.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:56.962 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:56.962 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:56.989 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install rbd-nbd 2026-02-22T22:36:57.296 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:57.380 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:57.380 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:57.380 INFO:teuthology.orchestra.run.trial078.stdout: Package Architecture Version Repository Size 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout:Installing: 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout:Transaction Summary 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout:================================================================================ 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout:Install 1 Package 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout:Total download size: 170 k 2026-02-22T22:36:57.381 INFO:teuthology.orchestra.run.trial078.stdout:Installed size: 490 k 2026-02-22T22:36:57.382 INFO:teuthology.orchestra.run.trial078.stdout:Downloading Packages: 2026-02-22T22:36:57.469 INFO:teuthology.orchestra.run.trial078.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.9 MB/s | 170 kB 00:00 2026-02-22T22:36:57.469 INFO:teuthology.orchestra.run.trial078.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:36:57.469 INFO:teuthology.orchestra.run.trial078.stdout:Total 1.9 MB/s | 170 kB 00:00 2026-02-22T22:36:57.469 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction check 2026-02-22T22:36:57.472 INFO:teuthology.orchestra.run.trial078.stdout:Transaction check succeeded. 2026-02-22T22:36:57.472 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction test 2026-02-22T22:36:57.504 INFO:teuthology.orchestra.run.trial078.stdout:Transaction test succeeded. 2026-02-22T22:36:57.504 INFO:teuthology.orchestra.run.trial078.stdout:Running transaction 2026-02-22T22:36:57.559 INFO:teuthology.orchestra.run.trial078.stdout: Preparing : 1/1 2026-02-22T22:36:57.576 INFO:teuthology.orchestra.run.trial078.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:57.663 INFO:teuthology.orchestra.run.trial078.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:57.723 INFO:teuthology.orchestra.run.trial078.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:36:57.723 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:57.723 INFO:teuthology.orchestra.run.trial078.stdout:Installed: 2026-02-22T22:36:57.723 INFO:teuthology.orchestra.run.trial078.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:36:57.723 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:36:57.723 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:57.752 DEBUG:teuthology.orchestra.run.trial078:> sudo yum -y install rbd-mirror 2026-02-22T22:36:58.073 INFO:teuthology.orchestra.run.trial078.stdout:Last metadata expiration check: 0:00:43 ago on Sun 22 Feb 2026 10:36:15 PM UTC. 2026-02-22T22:36:58.137 INFO:teuthology.orchestra.run.trial078.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:36:58.156 INFO:teuthology.orchestra.run.trial078.stdout:Dependencies resolved. 2026-02-22T22:36:58.156 INFO:teuthology.orchestra.run.trial078.stdout:Nothing to do. 2026-02-22T22:36:58.157 INFO:teuthology.orchestra.run.trial078.stdout:Complete! 2026-02-22T22:36:58.179 DEBUG:teuthology.parallel:result is None 2026-02-22T22:36:58.179 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:36:58.317 DEBUG:teuthology.orchestra.run.trial078:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T22:36:58.337 INFO:teuthology.orchestra.run.trial078.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T22:36:58.338 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T22:36:58.338 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T22:36:58.340 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:36:58.476 DEBUG:teuthology.orchestra.run.trial178:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T22:36:58.497 INFO:teuthology.orchestra.run.trial178.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T22:36:58.497 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T22:36:58.497 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T22:36:58.500 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-22T22:36:58.500 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:36:58.500 DEBUG:teuthology.orchestra.run.trial078:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T22:36:58.527 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:36:58.527 DEBUG:teuthology.orchestra.run.trial178:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T22:36:58.562 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-22T22:36:58.562 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:36:58.562 DEBUG:teuthology.orchestra.run.trial078:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T22:36:58.591 DEBUG:teuthology.orchestra.run.trial078:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T22:36:58.654 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:36:58.655 DEBUG:teuthology.orchestra.run.trial178:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T22:36:58.680 DEBUG:teuthology.orchestra.run.trial178:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T22:36:58.743 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-22T22:36:58.743 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:36:58.744 DEBUG:teuthology.orchestra.run.trial078:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T22:36:58.768 DEBUG:teuthology.orchestra.run.trial078:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T22:36:58.831 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:36:58.831 DEBUG:teuthology.orchestra.run.trial178:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T22:36:58.857 DEBUG:teuthology.orchestra.run.trial178:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T22:36:58.920 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-22T22:36:58.921 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:36:58.921 DEBUG:teuthology.orchestra.run.trial078:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T22:36:58.946 DEBUG:teuthology.orchestra.run.trial078:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T22:36:59.010 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:36:59.011 DEBUG:teuthology.orchestra.run.trial178:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T22:36:59.035 DEBUG:teuthology.orchestra.run.trial178:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T22:36:59.100 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T22:36:59.214 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-22T22:36:59.214 DEBUG:teuthology.orchestra.run.trial078:> sudo chmod 777 /var/log/ceph 2026-02-22T22:36:59.216 DEBUG:teuthology.orchestra.run.trial178:> sudo chmod 777 /var/log/ceph 2026-02-22T22:36:59.240 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-22T22:36:59.240 DEBUG:teuthology.orchestra.run.trial078:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T22:36:59.281 DEBUG:teuthology.orchestra.run.trial178:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T22:36:59.303 INFO:tasks.ceph:Creating extra log directories... 2026-02-22T22:36:59.303 DEBUG:teuthology.orchestra.run.trial078:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T22:36:59.344 DEBUG:teuthology.orchestra.run.trial178:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T22:36:59.387 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-22T22:36:59.387 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T22:36:59.387 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-bootstrap-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64259', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror_bootstrap.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial078.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDNDWZiJW4+mOO5x9eBXVo1SH1y+4ZVaepjXMZrz5ZUr76fZJVBweiNldE6oG8JquHGeOG7ACQ7DVVHDO5xMleY=', 'trial178.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMgjfWFN++E60nfU1X/ZHTxvvvI+OpzCzJpV7wLGORR/Gntf4Ene1CfaJ7Lliu5GSNruuGy7XXe+1TU5E4tsGbg='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_bootstrap.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_MIRROR_INSTANCES': '1', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T22:36:59.387 DEBUG:teuthology.orchestra.run.trial078:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T22:36:59.409 DEBUG:teuthology.orchestra.run.trial178:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T22:36:59.427 DEBUG:teuthology.orchestra.run.trial078:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:36:59.467 DEBUG:teuthology.orchestra.run.trial178:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:36:59.490 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:36:59.491 DEBUG:teuthology.orchestra.run.trial078:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T22:36:59.545 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T22:36:59.546 DEBUG:teuthology.orchestra.run.trial078:> stat /dev/vg_nvme/lv_1 2026-02-22T22:36:59.603 INFO:teuthology.orchestra.run.trial078.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T22:36:59.604 INFO:teuthology.orchestra.run.trial078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:36:59.604 INFO:teuthology.orchestra.run.trial078.stdout:Device: 6h/6d Inode: 1301 Links: 1 2026-02-22T22:36:59.604 INFO:teuthology.orchestra.run.trial078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:36:59.604 INFO:teuthology.orchestra.run.trial078.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:36:59.604 INFO:teuthology.orchestra.run.trial078.stdout:Access: 2026-02-22 22:36:56.805009834 +0000 2026-02-22T22:36:59.604 INFO:teuthology.orchestra.run.trial078.stdout:Modify: 2026-02-22 22:36:22.727955556 +0000 2026-02-22T22:36:59.604 INFO:teuthology.orchestra.run.trial078.stdout:Change: 2026-02-22 22:36:22.727955556 +0000 2026-02-22T22:36:59.604 INFO:teuthology.orchestra.run.trial078.stdout: Birth: 2026-02-22 22:36:22.727955556 +0000 2026-02-22T22:36:59.605 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T22:36:59.666 INFO:teuthology.orchestra.run.trial078.stderr:1+0 records in 2026-02-22T22:36:59.666 INFO:teuthology.orchestra.run.trial078.stderr:1+0 records out 2026-02-22T22:36:59.666 INFO:teuthology.orchestra.run.trial078.stderr:512 bytes copied, 0.0001599 s, 3.2 MB/s 2026-02-22T22:36:59.667 DEBUG:teuthology.orchestra.run.trial078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T22:36:59.726 DEBUG:teuthology.orchestra.run.trial078:> stat /dev/vg_nvme/lv_2 2026-02-22T22:36:59.781 INFO:teuthology.orchestra.run.trial078.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T22:36:59.781 INFO:teuthology.orchestra.run.trial078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:36:59.781 INFO:teuthology.orchestra.run.trial078.stdout:Device: 6h/6d Inode: 1309 Links: 1 2026-02-22T22:36:59.782 INFO:teuthology.orchestra.run.trial078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:36:59.782 INFO:teuthology.orchestra.run.trial078.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:36:59.782 INFO:teuthology.orchestra.run.trial078.stdout:Access: 2026-02-22 22:36:56.805009834 +0000 2026-02-22T22:36:59.782 INFO:teuthology.orchestra.run.trial078.stdout:Modify: 2026-02-22 22:36:22.728955558 +0000 2026-02-22T22:36:59.782 INFO:teuthology.orchestra.run.trial078.stdout:Change: 2026-02-22 22:36:22.728955558 +0000 2026-02-22T22:36:59.782 INFO:teuthology.orchestra.run.trial078.stdout: Birth: 2026-02-22 22:36:22.728955558 +0000 2026-02-22T22:36:59.782 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T22:36:59.844 INFO:teuthology.orchestra.run.trial078.stderr:1+0 records in 2026-02-22T22:36:59.844 INFO:teuthology.orchestra.run.trial078.stderr:1+0 records out 2026-02-22T22:36:59.844 INFO:teuthology.orchestra.run.trial078.stderr:512 bytes copied, 0.000145833 s, 3.5 MB/s 2026-02-22T22:36:59.845 DEBUG:teuthology.orchestra.run.trial078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T22:36:59.904 DEBUG:teuthology.orchestra.run.trial078:> stat /dev/vg_nvme/lv_3 2026-02-22T22:36:59.960 INFO:teuthology.orchestra.run.trial078.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T22:36:59.960 INFO:teuthology.orchestra.run.trial078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:36:59.960 INFO:teuthology.orchestra.run.trial078.stdout:Device: 6h/6d Inode: 1296 Links: 1 2026-02-22T22:36:59.961 INFO:teuthology.orchestra.run.trial078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:36:59.961 INFO:teuthology.orchestra.run.trial078.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:36:59.961 INFO:teuthology.orchestra.run.trial078.stdout:Access: 2026-02-22 22:36:56.805009834 +0000 2026-02-22T22:36:59.961 INFO:teuthology.orchestra.run.trial078.stdout:Modify: 2026-02-22 22:36:22.727955556 +0000 2026-02-22T22:36:59.961 INFO:teuthology.orchestra.run.trial078.stdout:Change: 2026-02-22 22:36:22.727955556 +0000 2026-02-22T22:36:59.961 INFO:teuthology.orchestra.run.trial078.stdout: Birth: 2026-02-22 22:36:22.727955556 +0000 2026-02-22T22:36:59.961 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T22:37:00.023 INFO:teuthology.orchestra.run.trial078.stderr:1+0 records in 2026-02-22T22:37:00.023 INFO:teuthology.orchestra.run.trial078.stderr:1+0 records out 2026-02-22T22:37:00.023 INFO:teuthology.orchestra.run.trial078.stderr:512 bytes copied, 0.000146223 s, 3.5 MB/s 2026-02-22T22:37:00.025 DEBUG:teuthology.orchestra.run.trial078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T22:37:00.082 DEBUG:teuthology.orchestra.run.trial078:> stat /dev/vg_nvme/lv_4 2026-02-22T22:37:00.138 INFO:teuthology.orchestra.run.trial078.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T22:37:00.138 INFO:teuthology.orchestra.run.trial078.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:37:00.139 INFO:teuthology.orchestra.run.trial078.stdout:Device: 6h/6d Inode: 1310 Links: 1 2026-02-22T22:37:00.139 INFO:teuthology.orchestra.run.trial078.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:37:00.139 INFO:teuthology.orchestra.run.trial078.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:37:00.139 INFO:teuthology.orchestra.run.trial078.stdout:Access: 2026-02-22 22:36:56.805009834 +0000 2026-02-22T22:37:00.139 INFO:teuthology.orchestra.run.trial078.stdout:Modify: 2026-02-22 22:36:22.728955558 +0000 2026-02-22T22:37:00.139 INFO:teuthology.orchestra.run.trial078.stdout:Change: 2026-02-22 22:36:22.728955558 +0000 2026-02-22T22:37:00.139 INFO:teuthology.orchestra.run.trial078.stdout: Birth: 2026-02-22 22:36:22.728955558 +0000 2026-02-22T22:37:00.139 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T22:37:00.201 INFO:teuthology.orchestra.run.trial078.stderr:1+0 records in 2026-02-22T22:37:00.202 INFO:teuthology.orchestra.run.trial078.stderr:1+0 records out 2026-02-22T22:37:00.202 INFO:teuthology.orchestra.run.trial078.stderr:512 bytes copied, 0.000137899 s, 3.7 MB/s 2026-02-22T22:37:00.203 DEBUG:teuthology.orchestra.run.trial078:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T22:37:00.262 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:37:00.262 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial078.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:37:00.262 INFO:tasks.ceph:Generating config... 2026-02-22T22:37:00.266 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T22:37:00.266 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.267 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.268 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.268 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.268 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.268 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.268 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.268 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.268 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T22:37:00.268 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T22:37:00.268 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T22:37:00.268 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T22:37:00.268 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T22:37:00.269 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T22:37:00.269 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T22:37:00.269 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] bluestore allocator = stupid 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T22:37:00.269 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T22:37:00.270 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T22:37:00.270 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T22:37:00.270 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T22:37:00.270 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T22:37:00.270 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T22:37:00.270 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T22:37:00.270 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-22T22:37:00.270 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-22T22:37:00.336 INFO:teuthology.orchestra.run.trial078.stdout:creating /etc/ceph/cluster1.keyring 2026-02-22T22:37:00.339 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-22T22:37:00.415 DEBUG:teuthology.orchestra.run.trial078:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:37:00.480 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.78')] 2026-02-22T22:37:00.481 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.78', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T22:37:00.482 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:00.482 DEBUG:teuthology.orchestra.run.trial078:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:37:00.535 DEBUG:teuthology.orchestra.run.trial078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.78 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:37:00.610 INFO:teuthology.orchestra.run.trial078.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:37:00.610 INFO:teuthology.orchestra.run.trial078.stdout:monmaptool: generated fsid 892254a3-cc2a-4241-8fd5-9cfdb608522a 2026-02-22T22:37:00.610 INFO:teuthology.orchestra.run.trial078.stdout:setting min_mon_release = quincy 2026-02-22T22:37:00.610 INFO:teuthology.orchestra.run.trial078.stdout:epoch 0 2026-02-22T22:37:00.610 INFO:teuthology.orchestra.run.trial078.stdout:fsid 892254a3-cc2a-4241-8fd5-9cfdb608522a 2026-02-22T22:37:00.610 INFO:teuthology.orchestra.run.trial078.stdout:last_changed 2026-02-22T22:37:00.607867+0000 2026-02-22T22:37:00.610 INFO:teuthology.orchestra.run.trial078.stdout:created 2026-02-22T22:37:00.607867+0000 2026-02-22T22:37:00.611 INFO:teuthology.orchestra.run.trial078.stdout:min_mon_release 17 (quincy) 2026-02-22T22:37:00.611 INFO:teuthology.orchestra.run.trial078.stdout:election_strategy: 1 2026-02-22T22:37:00.611 INFO:teuthology.orchestra.run.trial078.stdout:0: [v2:10.20.193.78:3300/0,v1:10.20.193.78:6789/0] mon.a 2026-02-22T22:37:00.611 INFO:teuthology.orchestra.run.trial078.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-22T22:37:00.613 DEBUG:teuthology.orchestra.run.trial078:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:37:00.668 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 892254a3-cc2a-4241-8fd5-9cfdb608522a... 2026-02-22T22:37:00.669 DEBUG:teuthology.orchestra.run.trial078:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T22:37:00.710 DEBUG:teuthology.orchestra.run.trial178:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T22:37:00.749 INFO:teuthology.orchestra.run.trial078.stdout:[global] 2026-02-22T22:37:00.749 INFO:teuthology.orchestra.run.trial078.stdout: chdir = "" 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: auth supported = cephx 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: filestore xattr use omap = true 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: mon clock drift allowed = 1.000 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: osd crush chooseleaf type = 0 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: auth debug = true 2026-02-22T22:37:00.750 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: ms die on old message = true 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: ms die on bug = true 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: mon pg warn max object skew = 0 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: osd pool default size = 2 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.751 INFO:teuthology.orchestra.run.trial078.stdout: mon osd allow primary affinity = true 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon osd allow pg remap = true 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on no sortbitwise = false 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on too few osds = false 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: mon_allow_pool_size_one = true 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.752 INFO:teuthology.orchestra.run.trial078.stdout: osd default data pool replay window = 5 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: mon allow pool delete = true 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: mon cluster log file level = debug 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: debug asserts on shutdown = true 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: mon health detail to clog = false 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: mon host = 10.20.193.78 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: mon client directed command retry = 5 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on pool no app = False 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: ms inject socket failures = 5000 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: fsid = 892254a3-cc2a-4241-8fd5-9cfdb608522a 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.753 INFO:teuthology.orchestra.run.trial078.stdout:[osd] 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: osd journal size = 100 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: osd scrub load threshold = 5.0 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: osd scrub max interval = 600 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: osd mclock skip benchmark = true 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: osd recover clone overlap = true 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: osd recovery max chunk = 1048576 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.754 INFO:teuthology.orchestra.run.trial078.stdout: osd debug shutdown = true 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: osd debug op order = true 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: osd debug verify stray on activate = true 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: osd debug trim objects = true 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: osd open classes on start = true 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: osd debug pg log writeout = true 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: osd map max advance = 10 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: journal zero on create = true 2026-02-22T22:37:00.755 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: filestore apply finisher threads = 3 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: bdev debug aio = true 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: osd debug misdirected ops = true 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: bdev async discard = True 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: bdev enable discard = True 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: bluestore allocator = stupid 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: bluestore block size = 96636764160 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: bluestore fsck on mount = True 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: debug bluefs = 1/20 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: debug bluestore = 1/20 2026-02-22T22:37:00.756 INFO:teuthology.orchestra.run.trial078.stdout: debug ms = 1 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: debug osd = 20 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: debug rocksdb = 4/10 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: mon osd full ratio = 0.9 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: osd objectstore = bluestore 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout:[mgr] 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: debug ms = 1 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: debug mgr = 20 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: debug mon = 20 2026-02-22T22:37:00.757 INFO:teuthology.orchestra.run.trial078.stdout: debug auth = 20 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: mgr/telemetry/nag = false 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout:[mon] 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: debug ms = 1 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: debug mon = 20 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: debug paxos = 20 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: debug auth = 20 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: mon data avail warn = 5 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: mon mgr mkfs grace = 240 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:37:00.758 INFO:teuthology.orchestra.run.trial078.stdout: mon osd reporter subtree level = osd 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: mon osd prime pg temp = true 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.759 INFO:teuthology.orchestra.run.trial078.stdout: # 1m isn't quite enough 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout:[client] 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: rgw cache enabled = true 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: rgw enable ops log = true 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: rgw enable usage log = true 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: debug rbd = 20 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: debug rbd_mirror = 15 2026-02-22T22:37:00.760 INFO:teuthology.orchestra.run.trial078.stdout: log to stderr = False 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout: rbd default features = 125 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout:[mon.a] 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.0] 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.1] 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.2] 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.761 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.3] 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.4] 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.5] 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.6] 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.762 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.764 INFO:teuthology.orchestra.run.trial178.stdout:[global] 2026-02-22T22:37:00.764 INFO:teuthology.orchestra.run.trial178.stdout: chdir = "" 2026-02-22T22:37:00.764 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: auth supported = cephx 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: filestore xattr use omap = true 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: mon clock drift allowed = 1.000 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: osd crush chooseleaf type = 0 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: auth debug = true 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: ms die on old message = true 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: ms die on bug = true 2026-02-22T22:37:00.765 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: mon pg warn max object skew = 0 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: osd pool default size = 2 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: mon osd allow primary affinity = true 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: mon osd allow pg remap = true 2026-02-22T22:37:00.766 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on no sortbitwise = false 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on too few osds = false 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: mon_allow_pool_size_one = true 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: osd default data pool replay window = 5 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.767 INFO:teuthology.orchestra.run.trial178.stdout: mon allow pool delete = true 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: mon cluster log file level = debug 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: debug asserts on shutdown = true 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: mon health detail to clog = false 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: mon host = 10.20.193.78 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: mon client directed command retry = 5 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on pool no app = False 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: ms inject socket failures = 5000 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: fsid = 892254a3-cc2a-4241-8fd5-9cfdb608522a 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout:[osd] 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: osd journal size = 100 2026-02-22T22:37:00.768 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd scrub load threshold = 5.0 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd scrub max interval = 600 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd mclock skip benchmark = true 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd recover clone overlap = true 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd recovery max chunk = 1048576 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd debug shutdown = true 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd debug op order = true 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd debug verify stray on activate = true 2026-02-22T22:37:00.769 INFO:teuthology.orchestra.run.trial178.stdout: osd debug trim objects = true 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: osd open classes on start = true 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: osd debug pg log writeout = true 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: osd map max advance = 10 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: journal zero on create = true 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:37:00.770 INFO:teuthology.orchestra.run.trial178.stdout: filestore apply finisher threads = 3 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: bdev debug aio = true 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: osd debug misdirected ops = true 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: bdev async discard = True 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: bdev enable discard = True 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: bluestore allocator = stupid 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: bluestore block size = 96636764160 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: bluestore fsck on mount = True 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: debug bluefs = 1/20 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: debug bluestore = 1/20 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: debug ms = 1 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: debug osd = 20 2026-02-22T22:37:00.771 INFO:teuthology.orchestra.run.trial178.stdout: debug rocksdb = 4/10 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: mon osd full ratio = 0.9 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: osd objectstore = bluestore 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout:[mgr] 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: debug ms = 1 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: debug mgr = 20 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: debug mon = 20 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: debug auth = 20 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:37:00.772 INFO:teuthology.orchestra.run.trial178.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: mgr/telemetry/nag = false 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout:[mon] 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: debug ms = 1 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: debug mon = 20 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: debug paxos = 20 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: debug auth = 20 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: mon data avail warn = 5 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: mon mgr mkfs grace = 240 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:37:00.773 INFO:teuthology.orchestra.run.trial178.stdout: mon osd reporter subtree level = osd 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: mon osd prime pg temp = true 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: # 1m isn't quite enough 2026-02-22T22:37:00.774 INFO:teuthology.orchestra.run.trial178.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout:[client] 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: rgw cache enabled = true 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: rgw enable ops log = true 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: rgw enable usage log = true 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: debug rbd = 20 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: debug rbd_mirror = 15 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: log to stderr = False 2026-02-22T22:37:00.775 INFO:teuthology.orchestra.run.trial178.stdout: rbd default features = 125 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout:[mon.a] 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.0] 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.1] 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.2] 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.3] 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.776 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.4] 2026-02-22T22:37:00.777 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.777 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.777 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.5] 2026-02-22T22:37:00.777 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.777 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.777 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.6] 2026-02-22T22:37:00.777 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:00.777 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:00.777 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-22T22:37:00.778 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-22T22:37:00.846 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T22:37:00.846 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:00.846 DEBUG:teuthology.orchestra.run.trial078:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-22T22:37:00.903 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:00.903 DEBUG:teuthology.orchestra.run.trial078:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-22T22:37:00.960 INFO:tasks.ceph:Sending monmap to node ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:37:00.960 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:00.960 DEBUG:teuthology.orchestra.run.trial078:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T22:37:00.960 DEBUG:teuthology.orchestra.run.trial078:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:37:01.032 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:01.032 DEBUG:teuthology.orchestra.run.trial078:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:37:01.089 INFO:tasks.ceph:Sending monmap to node ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:37:01.089 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:01.089 DEBUG:teuthology.orchestra.run.trial178:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T22:37:01.089 DEBUG:teuthology.orchestra.run.trial178:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:37:01.123 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:01.124 DEBUG:teuthology.orchestra.run.trial178:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:37:01.179 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T22:37:01.179 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T22:37:01.179 DEBUG:teuthology.orchestra.run.trial078:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T22:37:01.223 INFO:teuthology.orchestra.run.trial078.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T22:37:01.226 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T22:37:01.226 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T22:37:01.226 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-22T22:37:01.302 INFO:teuthology.orchestra.run.trial078.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-22T22:37:01.313 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T22:37:01.348 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T22:37:01.359 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T22:37:01.434 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T22:37:01.444 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T22:37:01.521 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T22:37:01.532 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T22:37:01.608 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T22:37:01.619 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T22:37:01.692 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T22:37:01.703 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T22:37:01.777 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T22:37:01.788 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T22:37:01.863 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T22:37:01.873 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T22:37:01.947 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T22:37:01.957 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T22:37:01.957 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial078.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:37:01.957 DEBUG:teuthology.orchestra.run.trial078:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-22T22:37:01.982 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:37:01.982 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-22T22:37:01.983 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:37:01.983 DEBUG:teuthology.orchestra.run.trial078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T22:37:02.049 INFO:teuthology.orchestra.run.trial078.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:37:02.049 INFO:teuthology.orchestra.run.trial078.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:37:02.049 INFO:teuthology.orchestra.run.trial078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:37:02.049 INFO:teuthology.orchestra.run.trial078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:37:02.050 INFO:teuthology.orchestra.run.trial078.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:37:02.050 INFO:teuthology.orchestra.run.trial078.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:37:02.050 INFO:teuthology.orchestra.run.trial078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:37:02.050 INFO:teuthology.orchestra.run.trial078.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:37:02.050 INFO:teuthology.orchestra.run.trial078.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:37:02.050 INFO:teuthology.orchestra.run.trial078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:37:02.621 INFO:teuthology.orchestra.run.trial078.stdout:Discarding blocks...Done. 2026-02-22T22:37:02.623 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial078.front.sepia.ceph.com -o noatime 2026-02-22T22:37:02.623 DEBUG:teuthology.orchestra.run.trial078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-22T22:37:02.691 DEBUG:teuthology.orchestra.run.trial078:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-22T22:37:02.759 DEBUG:teuthology.orchestra.run.trial078:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-22T22:37:02.824 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:37:02.825 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-22T22:37:02.825 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:37:02.825 DEBUG:teuthology.orchestra.run.trial078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T22:37:02.891 INFO:teuthology.orchestra.run.trial078.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:37:02.891 INFO:teuthology.orchestra.run.trial078.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:37:02.891 INFO:teuthology.orchestra.run.trial078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:37:02.891 INFO:teuthology.orchestra.run.trial078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:37:02.891 INFO:teuthology.orchestra.run.trial078.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:37:02.891 INFO:teuthology.orchestra.run.trial078.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:37:02.891 INFO:teuthology.orchestra.run.trial078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:37:02.892 INFO:teuthology.orchestra.run.trial078.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:37:02.892 INFO:teuthology.orchestra.run.trial078.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:37:02.892 INFO:teuthology.orchestra.run.trial078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:37:03.453 INFO:teuthology.orchestra.run.trial078.stdout:Discarding blocks...Done. 2026-02-22T22:37:03.455 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial078.front.sepia.ceph.com -o noatime 2026-02-22T22:37:03.455 DEBUG:teuthology.orchestra.run.trial078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-22T22:37:03.523 DEBUG:teuthology.orchestra.run.trial078:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-22T22:37:03.591 DEBUG:teuthology.orchestra.run.trial078:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-22T22:37:03.657 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:37:03.657 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-22T22:37:03.657 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:37:03.657 DEBUG:teuthology.orchestra.run.trial078:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T22:37:03.724 INFO:teuthology.orchestra.run.trial078.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:37:03.724 INFO:teuthology.orchestra.run.trial078.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:37:03.724 INFO:teuthology.orchestra.run.trial078.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:37:03.724 INFO:teuthology.orchestra.run.trial078.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:37:03.724 INFO:teuthology.orchestra.run.trial078.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:37:03.725 INFO:teuthology.orchestra.run.trial078.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:37:03.725 INFO:teuthology.orchestra.run.trial078.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:37:03.725 INFO:teuthology.orchestra.run.trial078.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:37:03.725 INFO:teuthology.orchestra.run.trial078.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:37:03.725 INFO:teuthology.orchestra.run.trial078.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:37:04.282 INFO:teuthology.orchestra.run.trial078.stdout:Discarding blocks...Done. 2026-02-22T22:37:04.284 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial078.front.sepia.ceph.com -o noatime 2026-02-22T22:37:04.284 DEBUG:teuthology.orchestra.run.trial078:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-22T22:37:04.353 DEBUG:teuthology.orchestra.run.trial078:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-22T22:37:04.420 DEBUG:teuthology.orchestra.run.trial078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:37:04.498 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:04.495+0000 7fe8dc810780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-22T22:37:04.499 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:04.496+0000 7fe8dc810780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-22T22:37:04.499 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:04.496+0000 7fe8dc810780 -1 bdev(0x560e7107f800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-22T22:37:04.499 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:04.496+0000 7fe8dc810780 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-22T22:37:08.384 DEBUG:teuthology.orchestra.run.trial078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-22T22:37:08.451 DEBUG:teuthology.orchestra.run.trial078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:37:08.531 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:08.528+0000 7fa5dfec8780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-22T22:37:08.531 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:08.528+0000 7fa5dfec8780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-22T22:37:08.531 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:08.528+0000 7fa5dfec8780 -1 bdev(0x56011fe8d800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-22T22:37:08.532 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:08.528+0000 7fa5dfec8780 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-22T22:37:12.422 DEBUG:teuthology.orchestra.run.trial078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-22T22:37:12.488 DEBUG:teuthology.orchestra.run.trial078:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:37:12.567 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:12.564+0000 7f8d4602e780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-22T22:37:12.567 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:12.564+0000 7f8d4602e780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-22T22:37:12.568 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:12.564+0000 7f8d4602e780 -1 bdev(0x561e841a5800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-22T22:37:12.568 INFO:teuthology.orchestra.run.trial078.stderr:2026-02-22T22:37:12.564+0000 7f8d4602e780 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-22T22:37:16.458 DEBUG:teuthology.orchestra.run.trial078:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-22T22:37:16.524 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T22:37:16.524 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:16.524 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-22T22:37:16.588 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:16.588 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-22T22:37:16.654 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:16.655 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-22T22:37:16.718 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:16.719 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-22T22:37:16.783 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:16.783 DEBUG:teuthology.orchestra.run.trial078:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-22T22:37:16.838 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:16.838 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-22T22:37:16.855 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:16.855 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-22T22:37:16.912 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:16.913 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-22T22:37:16.968 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:16.968 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-22T22:37:17.024 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:17.024 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-22T22:37:17.080 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:17.080 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-22T22:37:17.137 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:17.137 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-22T22:37:17.193 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:17.193 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-22T22:37:17.249 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T22:37:17.249 DEBUG:teuthology.orchestra.run.trial078:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[mgr.x] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtpl3w3DRAAGnMW5cHDS1aaAUZLLm7Yvg== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[osd.0] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQAQhZtpX/SaHRAAPl1T2jylvQ4itDyCsJoVdg== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[osd.1] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQAUhZtpIhKNHxAAQh+g1mIfE3CQA8yQJkn8pg== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[osd.2] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQAYhZtpQIC0IRAAmvPCnQb+B2xYZRyL90VclA== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[client.0] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtppb7tERAA55wbsh4CCLHOq9IAl94hpg== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtpRR2uFBAALH/abVpa4XAbhBMSSfYGDg== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.0] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtpbtvDGRAAMoT1/e9FgKJZ6s0QmmitVg== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.1] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtp/b/6HhAAHxFTXhm3wQEGf7H6QOTk+A== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.2] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtp/bYqJBAARWCLeV/3o4JxeZusEyGjkw== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.3] 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtprJotKRAAxxhsJDyFX3DNyj+CWoS2Rw== 2026-02-22T22:37:17.272 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.4] 2026-02-22T22:37:17.273 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtpXD04LhAAnGaH2U5HZe/z7Q4XZhK4Hw== 2026-02-22T22:37:17.273 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.5] 2026-02-22T22:37:17.273 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtpgpdeMxAAebAPohhR7T8wK2Jd9w1ETw== 2026-02-22T22:37:17.273 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.6] 2026-02-22T22:37:17.273 INFO:teuthology.orchestra.run.trial078.stdout: key = AQANhZtpGalaOBAAaiN4kCj4TU4V681+21JqPg== 2026-02-22T22:37:17.273 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T22:37:17.353 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:37:17.434 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:37:17.514 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:37:17.594 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:17.676 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:17.757 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:17.838 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:17.921 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:18.004 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:18.085 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:18.164 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:18.248 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:18.329 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T22:37:18.329 DEBUG:teuthology.orchestra.run.trial078:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-22T22:37:18.393 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-22T22:37:18.480 DEBUG:teuthology.orchestra.run.trial078:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-22T22:37:18.504 DEBUG:teuthology.orchestra.run.trial078:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:37:18.559 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-22T22:37:18.559 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-22T22:37:18.559 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-22T22:37:18.601 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-22T22:37:18.601 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-22T22:37:18.601 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-22T22:37:18.602 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-22T22:37:18.604 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-22T22:37:18.604 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T22:37:18.604 DEBUG:teuthology.orchestra.run.trial078:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-22T22:37:18.706 INFO:teuthology.orchestra.run.trial078.stderr:adjusted tunables profile to default 2026-02-22T22:37:18.714 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T22:37:18.714 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-22T22:37:18.715 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:18.715 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-22T22:37:18.740 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:18.741 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-22T22:37:18.805 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:18.806 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-22T22:37:18.869 DEBUG:teuthology.orchestra.run.trial078:> sudo ceph --cluster cluster1 osd new b684e760-514f-4f15-bf12-40cad6d17463 0 2026-02-22T22:37:19.013 INFO:teuthology.orchestra.run.trial078.stdout:0 2026-02-22T22:37:19.023 DEBUG:teuthology.orchestra.run.trial078:> sudo ceph --cluster cluster1 osd new 7f3bfea4-7d3e-4178-a8d2-e6d4cf6fc32c 1 2026-02-22T22:37:19.128 INFO:teuthology.orchestra.run.trial078.stdout:1 2026-02-22T22:37:19.139 DEBUG:teuthology.orchestra.run.trial078:> sudo ceph --cluster cluster1 osd new 7b7e764e-186f-44d7-9826-54a235d28187 2 2026-02-22T22:37:19.213 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.210+0000 7f87b042d140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T22:37:19.242 INFO:teuthology.orchestra.run.trial078.stdout:2 2026-02-22T22:37:19.251 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-22T22:37:19.251 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-22T22:37:19.253 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-22T22:37:19.253 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-22T22:37:19.253 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-22T22:37:19.255 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-22T22:37:19.255 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-22T22:37:19.255 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-22T22:37:19.256 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-22T22:37:19.257 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:37:19.340 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.337+0000 7f87b042d140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T22:37:19.359 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:19.359 INFO:teuthology.orchestra.run.trial078.stdout:{"epoch":5,"fsid":"892254a3-cc2a-4241-8fd5-9cfdb608522a","created":"2026-02-22T22:37:18.663727+0000","modified":"2026-02-22T22:37:19.240135+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:37:19.240135+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"b684e760-514f-4f15-bf12-40cad6d17463","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"7f3bfea4-7d3e-4178-a8d2-e6d4cf6fc32c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"7b7e764e-186f-44d7-9826-54a235d28187","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:37:19.367 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-22T22:37:19.368 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T22:37:19.368 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.364+0000 7f87b042d140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T22:37:19.401 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.398+0000 7f87b042d140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T22:37:19.424 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.421+0000 7f87b042d140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T22:37:19.450 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.447+0000 7f87b042d140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T22:37:19.506 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.503+0000 7f87b042d140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T22:37:19.567 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T22:37:19.567 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T22:37:19.567 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr: from numpy import show_config as show_numpy_config 2026-02-22T22:37:19.570 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.567+0000 7f87b042d140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T22:37:19.595 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.593+0000 7f87b042d140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T22:37:19.651 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.648+0000 7f87b042d140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T22:37:19.670 DEBUG:teuthology.orchestra.run.trial078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:37:19.694 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.691+0000 7f87b042d140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T22:37:19.720 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.717+0000 7f87b042d140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T22:37:19.745 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.742+0000 7f87b042d140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T22:37:19.763 INFO:teuthology.misc.health.trial078.stdout: 2026-02-22T22:37:19.763 INFO:teuthology.misc.health.trial078.stdout:{"epoch":5,"fsid":"892254a3-cc2a-4241-8fd5-9cfdb608522a","created":"2026-02-22T22:37:18.663727+0000","modified":"2026-02-22T22:37:19.240135+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:37:19.240135+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"b684e760-514f-4f15-bf12-40cad6d17463","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"7f3bfea4-7d3e-4178-a8d2-e6d4cf6fc32c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"7b7e764e-186f-44d7-9826-54a235d28187","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:37:19.768 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:37:19.770 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.768+0000 7f87b042d140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T22:37:19.793 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.791+0000 7f87b042d140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T22:37:19.988 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:19.985+0000 7f87b042d140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T22:37:20.043 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.040+0000 7f87b042d140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T22:37:20.065 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.063+0000 7f87b042d140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T22:37:20.088 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.086+0000 7f87b042d140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T22:37:20.141 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.139+0000 7f87b042d140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:37:20.162 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.160+0000 7f87b042d140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T22:37:20.214 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.212+0000 7f87b042d140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:37:20.611 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.608+0000 7f87b042d140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T22:37:20.633 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.630+0000 7f87b042d140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T22:37:20.938 INFO:tasks.ceph.cluster1.osd.1.trial078.stderr:2026-02-22T22:37:20.935+0000 7f579ba11780 -1 Falling back to public interface 2026-02-22T22:37:20.974 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:20.971+0000 7f87b042d140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T22:37:21.177 INFO:tasks.ceph.cluster1.osd.0.trial078.stderr:2026-02-22T22:37:21.174+0000 7f6abb20e780 -1 Falling back to public interface 2026-02-22T22:37:21.197 INFO:tasks.ceph.cluster1.osd.2.trial078.stderr:2026-02-22T22:37:21.194+0000 7fa6fd027780 -1 Falling back to public interface 2026-02-22T22:37:21.979 INFO:tasks.ceph.cluster1.mgr.x.trial078.stderr:2026-02-22T22:37:21.976+0000 7f86cdbb9640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T22:37:25.466 INFO:tasks.ceph.cluster1.osd.1.trial078.stderr:2026-02-22T22:37:25.463+0000 7f579ba11780 -1 osd.1 0 log_to_monitors true 2026-02-22T22:37:25.650 INFO:tasks.ceph.cluster1.osd.0.trial078.stderr:2026-02-22T22:37:25.646+0000 7f6abb20e780 -1 osd.0 0 log_to_monitors true 2026-02-22T22:37:25.701 INFO:tasks.ceph.cluster1.osd.2.trial078.stderr:2026-02-22T22:37:25.697+0000 7fa6fd027780 -1 osd.2 0 log_to_monitors true 2026-02-22T22:37:26.071 DEBUG:teuthology.orchestra.run.trial078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:37:26.222 INFO:teuthology.misc.health.trial078.stdout: 2026-02-22T22:37:26.222 INFO:teuthology.misc.health.trial078.stdout:{"epoch":6,"fsid":"892254a3-cc2a-4241-8fd5-9cfdb608522a","created":"2026-02-22T22:37:18.663727+0000","modified":"2026-02-22T22:37:25.517331+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:37:19.240135+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"b684e760-514f-4f15-bf12-40cad6d17463","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"7f3bfea4-7d3e-4178-a8d2-e6d4cf6fc32c","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"7b7e764e-186f-44d7-9826-54a235d28187","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:37:26.229 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:37:26.532 INFO:tasks.ceph.cluster1.osd.1.trial078.stderr:2026-02-22T22:37:26.529+0000 7f57973b2640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T22:37:26.534 INFO:tasks.ceph.cluster1.osd.1.trial078.stderr:2026-02-22T22:37:26.530+0000 7f578f1e5640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:37:27.539 INFO:tasks.ceph.cluster1.osd.0.trial078.stderr:2026-02-22T22:37:27.536+0000 7f6ab6bd2640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T22:37:27.541 INFO:tasks.ceph.cluster1.osd.0.trial078.stderr:2026-02-22T22:37:27.538+0000 7f6aaea05640 -1 osd.0 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:37:27.546 INFO:tasks.ceph.cluster1.osd.2.trial078.stderr:2026-02-22T22:37:27.543+0000 7fa6f8f92640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T22:37:27.548 INFO:tasks.ceph.cluster1.osd.2.trial078.stderr:2026-02-22T22:37:27.544+0000 7fa6f0dc5640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:37:32.532 DEBUG:teuthology.orchestra.run.trial078:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:37:32.685 INFO:teuthology.misc.health.trial078.stdout: 2026-02-22T22:37:32.686 INFO:teuthology.misc.health.trial078.stdout:{"epoch":12,"fsid":"892254a3-cc2a-4241-8fd5-9cfdb608522a","created":"2026-02-22T22:37:18.663727+0000","modified":"2026-02-22T22:37:31.529181+0000","last_up_change":"2026-02-22T22:37:28.524307+0000","last_in_change":"2026-02-22T22:37:19.240135+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:37:28.991462+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b684e760-514f-4f15-bf12-40cad6d17463","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6810","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6811","nonce":3604302546}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6812","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6813","nonce":3604302546}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6816","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6817","nonce":3604302546}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6814","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6815","nonce":3604302546}]},"public_addr":"10.20.193.78:6811/3604302546","cluster_addr":"10.20.193.78:6813/3604302546","heartbeat_back_addr":"10.20.193.78:6817/3604302546","heartbeat_front_addr":"10.20.193.78:6815/3604302546","state":["exists","up"]},{"osd":1,"uuid":"7f3bfea4-7d3e-4178-a8d2-e6d4cf6fc32c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6800","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6801","nonce":1380231211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6802","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6803","nonce":1380231211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6806","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6807","nonce":1380231211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6804","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6805","nonce":1380231211}]},"public_addr":"10.20.193.78:6801/1380231211","cluster_addr":"10.20.193.78:6803/1380231211","heartbeat_back_addr":"10.20.193.78:6807/1380231211","heartbeat_front_addr":"10.20.193.78:6805/1380231211","state":["exists","up"]},{"osd":2,"uuid":"7b7e764e-186f-44d7-9826-54a235d28187","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6818","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6819","nonce":2777796395}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6820","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6821","nonce":2777796395}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6824","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6825","nonce":2777796395}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6822","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6823","nonce":2777796395}]},"public_addr":"10.20.193.78:6819/2777796395","cluster_addr":"10.20.193.78:6821/2777796395","heartbeat_back_addr":"10.20.193.78:6825/2777796395","heartbeat_front_addr":"10.20.193.78:6823/2777796395","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.613807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.475169+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.712892+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:37:32.693 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T22:37:32.693 INFO:tasks.ceph:Creating RBD pool 2026-02-22T22:37:32.693 DEBUG:teuthology.orchestra.run.trial078:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-22T22:37:33.537 INFO:teuthology.orchestra.run.trial078.stderr:pool 'rbd' created 2026-02-22T22:37:33.549 DEBUG:teuthology.orchestra.run.trial078:> rbd --cluster cluster1 pool init rbd 2026-02-22T22:37:36.552 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-22T22:37:36.553 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-22T22:37:36.553 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-22T22:37:36.554 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-22T22:37:36.554 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T22:37:36.737 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:36.746 INFO:teuthology.orchestra.run.trial078.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6808","nonce":3737221826},{"type":"v1","addr":"10.20.193.78:6809","nonce":3737221826}]},"active_addr":"10.20.193.78:6809/3737221826","active_change":"2026-02-22T22:37:20.973459+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.78:0","nonce":883083744}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.78:0","nonce":3533025221}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.78:0","nonce":2498456794}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.78:0","nonce":4044929629}]}]} 2026-02-22T22:37:36.750 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-22T22:37:36.750 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-22T22:37:36.751 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:37:36.916 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:36.916 INFO:teuthology.orchestra.run.trial078.stdout:{"epoch":16,"fsid":"892254a3-cc2a-4241-8fd5-9cfdb608522a","created":"2026-02-22T22:37:18.663727+0000","modified":"2026-02-22T22:37:36.540330+0000","last_up_change":"2026-02-22T22:37:28.524307+0000","last_in_change":"2026-02-22T22:37:19.240135+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:37:28.991462+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:37:32.851019+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b684e760-514f-4f15-bf12-40cad6d17463","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6810","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6811","nonce":3604302546}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6812","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6813","nonce":3604302546}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6816","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6817","nonce":3604302546}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6814","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6815","nonce":3604302546}]},"public_addr":"10.20.193.78:6811/3604302546","cluster_addr":"10.20.193.78:6813/3604302546","heartbeat_back_addr":"10.20.193.78:6817/3604302546","heartbeat_front_addr":"10.20.193.78:6815/3604302546","state":["exists","up"]},{"osd":1,"uuid":"7f3bfea4-7d3e-4178-a8d2-e6d4cf6fc32c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6800","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6801","nonce":1380231211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6802","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6803","nonce":1380231211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6806","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6807","nonce":1380231211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6804","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6805","nonce":1380231211}]},"public_addr":"10.20.193.78:6801/1380231211","cluster_addr":"10.20.193.78:6803/1380231211","heartbeat_back_addr":"10.20.193.78:6807/1380231211","heartbeat_front_addr":"10.20.193.78:6805/1380231211","state":["exists","up"]},{"osd":2,"uuid":"7b7e764e-186f-44d7-9826-54a235d28187","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6818","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6819","nonce":2777796395}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6820","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6821","nonce":2777796395}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6824","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6825","nonce":2777796395}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6822","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6823","nonce":2777796395}]},"public_addr":"10.20.193.78:6819/2777796395","cluster_addr":"10.20.193.78:6821/2777796395","heartbeat_back_addr":"10.20.193.78:6825/2777796395","heartbeat_front_addr":"10.20.193.78:6823/2777796395","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.613807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.475169+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.712892+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:37:36.924 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-22T22:37:36.925 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:37:37.084 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:37.085 INFO:teuthology.orchestra.run.trial078.stdout:{"epoch":16,"fsid":"892254a3-cc2a-4241-8fd5-9cfdb608522a","created":"2026-02-22T22:37:18.663727+0000","modified":"2026-02-22T22:37:36.540330+0000","last_up_change":"2026-02-22T22:37:28.524307+0000","last_in_change":"2026-02-22T22:37:19.240135+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:37:28.991462+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:37:32.851019+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b684e760-514f-4f15-bf12-40cad6d17463","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6810","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6811","nonce":3604302546}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6812","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6813","nonce":3604302546}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6816","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6817","nonce":3604302546}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6814","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6815","nonce":3604302546}]},"public_addr":"10.20.193.78:6811/3604302546","cluster_addr":"10.20.193.78:6813/3604302546","heartbeat_back_addr":"10.20.193.78:6817/3604302546","heartbeat_front_addr":"10.20.193.78:6815/3604302546","state":["exists","up"]},{"osd":1,"uuid":"7f3bfea4-7d3e-4178-a8d2-e6d4cf6fc32c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6800","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6801","nonce":1380231211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6802","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6803","nonce":1380231211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6806","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6807","nonce":1380231211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6804","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6805","nonce":1380231211}]},"public_addr":"10.20.193.78:6801/1380231211","cluster_addr":"10.20.193.78:6803/1380231211","heartbeat_back_addr":"10.20.193.78:6807/1380231211","heartbeat_front_addr":"10.20.193.78:6805/1380231211","state":["exists","up"]},{"osd":2,"uuid":"7b7e764e-186f-44d7-9826-54a235d28187","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6818","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6819","nonce":2777796395}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6820","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6821","nonce":2777796395}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6824","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6825","nonce":2777796395}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6822","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6823","nonce":2777796395}]},"public_addr":"10.20.193.78:6819/2777796395","cluster_addr":"10.20.193.78:6821/2777796395","heartbeat_back_addr":"10.20.193.78:6825/2777796395","heartbeat_front_addr":"10.20.193.78:6823/2777796395","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.613807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.475169+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.712892+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:37:37.094 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-22T22:37:37.095 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-22T22:37:37.095 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-22T22:37:37.182 INFO:teuthology.orchestra.run.trial078.stdout:34359738371 2026-02-22T22:37:37.183 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:37:37.184 INFO:teuthology.orchestra.run.trial078.stdout:38654705669 2026-02-22T22:37:37.184 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:37:37.185 INFO:teuthology.orchestra.run.trial078.stdout:38654705668 2026-02-22T22:37:37.186 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:37:37.385 INFO:teuthology.orchestra.run.trial078.stdout:38654705668 2026-02-22T22:37:37.385 INFO:teuthology.orchestra.run.trial078.stdout:34359738370 2026-02-22T22:37:37.389 INFO:teuthology.orchestra.run.trial078.stdout:38654705667 2026-02-22T22:37:37.396 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-02-22T22:37:37.396 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705668 for osd.0 2026-02-22T22:37:37.399 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T22:37:38.396 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:37:38.397 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:37:38.399 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:37:38.558 INFO:teuthology.orchestra.run.trial078.stdout:34359738370 2026-02-22T22:37:38.560 INFO:teuthology.orchestra.run.trial078.stdout:38654705668 2026-02-22T22:37:38.562 INFO:teuthology.orchestra.run.trial078.stdout:38654705667 2026-02-22T22:37:38.568 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.1 2026-02-22T22:37:38.571 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705668 for osd.0 2026-02-22T22:37:38.573 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T22:37:39.570 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:37:39.571 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:37:39.574 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:37:39.732 INFO:teuthology.orchestra.run.trial078.stdout:34359738372 2026-02-22T22:37:39.736 INFO:teuthology.orchestra.run.trial078.stdout:38654705669 2026-02-22T22:37:39.741 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738372 for osd.1 2026-02-22T22:37:39.742 DEBUG:teuthology.parallel:result is None 2026-02-22T22:37:39.750 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705669 got 38654705669 for osd.0 2026-02-22T22:37:39.750 DEBUG:teuthology.parallel:result is None 2026-02-22T22:37:39.770 INFO:teuthology.orchestra.run.trial078.stdout:38654705669 2026-02-22T22:37:39.780 INFO:tasks.ceph.ceph_manager.cluster1:need seq 38654705668 got 38654705669 for osd.2 2026-02-22T22:37:39.780 DEBUG:teuthology.parallel:result is None 2026-02-22T22:37:39.780 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:37:39.781 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:37:39.976 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:39.976 INFO:teuthology.orchestra.run.trial078.stderr:dumped all 2026-02-22T22:37:39.984 INFO:teuthology.orchestra.run.trial078.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:37:38.977819+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82484,"kb_used_data":1576,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033036,"statfs":{"total":289910292480,"available":289825828864,"internally_reserved":0,"allocated":1613824,"data_stored":1302919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":82767188},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.441823"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.547270+0000","last_change":"2026-02-22T22:37:36.547270+0000","last_active":"2026-02-22T22:37:36.547270+0000","last_peered":"2026-02-22T22:37:36.547270+0000","last_clean":"2026-02-22T22:37:36.547270+0000","last_became_active":"2026-02-22T22:37:34.543281+0000","last_became_peered":"2026-02-22T22:37:34.543281+0000","last_unstale":"2026-02-22T22:37:36.547270+0000","last_undegraded":"2026-02-22T22:37:36.547270+0000","last_fullsized":"2026-02-22T22:37:36.547270+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:45:13.292837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077929400000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.547157+0000","last_change":"2026-02-22T22:37:36.547157+0000","last_active":"2026-02-22T22:37:36.547157+0000","last_peered":"2026-02-22T22:37:36.547157+0000","last_clean":"2026-02-22T22:37:36.547157+0000","last_became_active":"2026-02-22T22:37:34.542879+0000","last_became_peered":"2026-02-22T22:37:34.542879+0000","last_unstale":"2026-02-22T22:37:36.547157+0000","last_undegraded":"2026-02-22T22:37:36.547157+0000","last_fullsized":"2026-02-22T22:37:36.547157+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:42:20.635275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070406400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.546965+0000","last_change":"2026-02-22T22:37:36.546965+0000","last_active":"2026-02-22T22:37:36.546965+0000","last_peered":"2026-02-22T22:37:36.546965+0000","last_clean":"2026-02-22T22:37:36.546965+0000","last_became_active":"2026-02-22T22:37:34.543262+0000","last_became_peered":"2026-02-22T22:37:34.543262+0000","last_unstale":"2026-02-22T22:37:36.546965+0000","last_undegraded":"2026-02-22T22:37:36.546965+0000","last_fullsized":"2026-02-22T22:37:36.546965+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:37:36.614016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059131299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.547113+0000","last_change":"2026-02-22T22:37:36.547113+0000","last_active":"2026-02-22T22:37:36.547113+0000","last_peered":"2026-02-22T22:37:36.547113+0000","last_clean":"2026-02-22T22:37:36.547113+0000","last_became_active":"2026-02-22T22:37:34.543245+0000","last_became_peered":"2026-02-22T22:37:34.543245+0000","last_unstale":"2026-02-22T22:37:36.547113+0000","last_undegraded":"2026-02-22T22:37:36.547113+0000","last_fullsized":"2026-02-22T22:37:36.547113+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:06:29.211864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068412599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.545375+0000","last_change":"2026-02-22T22:37:36.545375+0000","last_active":"2026-02-22T22:37:36.545375+0000","last_peered":"2026-02-22T22:37:36.545375+0000","last_clean":"2026-02-22T22:37:36.545375+0000","last_became_active":"2026-02-22T22:37:34.543078+0000","last_became_peered":"2026-02-22T22:37:34.543078+0000","last_unstale":"2026-02-22T22:37:36.545375+0000","last_undegraded":"2026-02-22T22:37:36.545375+0000","last_fullsized":"2026-02-22T22:37:36.545375+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:44:07.853738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066853699999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.542745+0000","last_change":"2026-02-22T22:37:36.542745+0000","last_active":"2026-02-22T22:37:36.542745+0000","last_peered":"2026-02-22T22:37:36.542745+0000","last_clean":"2026-02-22T22:37:36.542745+0000","last_became_active":"2026-02-22T22:37:34.542908+0000","last_became_peered":"2026-02-22T22:37:34.542908+0000","last_unstale":"2026-02-22T22:37:36.542745+0000","last_undegraded":"2026-02-22T22:37:36.542745+0000","last_fullsized":"2026-02-22T22:37:36.542745+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:47:13.400204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043051299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.542742+0000","last_change":"2026-02-22T22:37:36.542742+0000","last_active":"2026-02-22T22:37:36.542742+0000","last_peered":"2026-02-22T22:37:36.542742+0000","last_clean":"2026-02-22T22:37:36.542742+0000","last_became_active":"2026-02-22T22:37:34.542961+0000","last_became_peered":"2026-02-22T22:37:34.542961+0000","last_unstale":"2026-02-22T22:37:36.542742+0000","last_undegraded":"2026-02-22T22:37:36.542742+0000","last_fullsized":"2026-02-22T22:37:36.542742+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:14.729623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000344994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.546970+0000","last_change":"2026-02-22T22:37:36.546970+0000","last_active":"2026-02-22T22:37:36.546970+0000","last_peered":"2026-02-22T22:37:36.546970+0000","last_clean":"2026-02-22T22:37:36.546970+0000","last_became_active":"2026-02-22T22:37:34.542584+0000","last_became_peered":"2026-02-22T22:37:34.542584+0000","last_unstale":"2026-02-22T22:37:36.546970+0000","last_undegraded":"2026-02-22T22:37:36.546970+0000","last_fullsized":"2026-02-22T22:37:36.546970+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:43:40.952490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060424800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.546580+0000","last_change":"2026-02-22T22:37:30.530613+0000","last_active":"2026-02-22T22:37:36.546580+0000","last_peered":"2026-02-22T22:37:36.546580+0000","last_clean":"2026-02-22T22:37:36.546580+0000","last_became_active":"2026-02-22T22:37:30.530398+0000","last_became_peered":"2026-02-22T22:37:30.530398+0000","last_unstale":"2026-02-22T22:37:36.546580+0000","last_undegraded":"2026-02-22T22:37:36.546580+0000","last_fullsized":"2026-02-22T22:37:36.546580+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_clean_scrub_stamp":"2026-02-22T22:37:29.526086+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:41:58.671728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705669,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27104,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344736,"statfs":{"total":96636764160,"available":96609009664,"internally_reserved":0,"allocated":139264,"data_stored":40715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27589064},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:37:39.986 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:37:40.140 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:40.141 INFO:teuthology.orchestra.run.trial078.stderr:dumped all 2026-02-22T22:37:40.148 INFO:teuthology.orchestra.run.trial078.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:37:38.977819+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82484,"kb_used_data":1576,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033036,"statfs":{"total":289910292480,"available":289825828864,"internally_reserved":0,"allocated":1613824,"data_stored":1302919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":82767188},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.441823"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.547270+0000","last_change":"2026-02-22T22:37:36.547270+0000","last_active":"2026-02-22T22:37:36.547270+0000","last_peered":"2026-02-22T22:37:36.547270+0000","last_clean":"2026-02-22T22:37:36.547270+0000","last_became_active":"2026-02-22T22:37:34.543281+0000","last_became_peered":"2026-02-22T22:37:34.543281+0000","last_unstale":"2026-02-22T22:37:36.547270+0000","last_undegraded":"2026-02-22T22:37:36.547270+0000","last_fullsized":"2026-02-22T22:37:36.547270+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:45:13.292837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077929400000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.547157+0000","last_change":"2026-02-22T22:37:36.547157+0000","last_active":"2026-02-22T22:37:36.547157+0000","last_peered":"2026-02-22T22:37:36.547157+0000","last_clean":"2026-02-22T22:37:36.547157+0000","last_became_active":"2026-02-22T22:37:34.542879+0000","last_became_peered":"2026-02-22T22:37:34.542879+0000","last_unstale":"2026-02-22T22:37:36.547157+0000","last_undegraded":"2026-02-22T22:37:36.547157+0000","last_fullsized":"2026-02-22T22:37:36.547157+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:42:20.635275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070406400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.546965+0000","last_change":"2026-02-22T22:37:36.546965+0000","last_active":"2026-02-22T22:37:36.546965+0000","last_peered":"2026-02-22T22:37:36.546965+0000","last_clean":"2026-02-22T22:37:36.546965+0000","last_became_active":"2026-02-22T22:37:34.543262+0000","last_became_peered":"2026-02-22T22:37:34.543262+0000","last_unstale":"2026-02-22T22:37:36.546965+0000","last_undegraded":"2026-02-22T22:37:36.546965+0000","last_fullsized":"2026-02-22T22:37:36.546965+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:37:36.614016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059131299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.547113+0000","last_change":"2026-02-22T22:37:36.547113+0000","last_active":"2026-02-22T22:37:36.547113+0000","last_peered":"2026-02-22T22:37:36.547113+0000","last_clean":"2026-02-22T22:37:36.547113+0000","last_became_active":"2026-02-22T22:37:34.543245+0000","last_became_peered":"2026-02-22T22:37:34.543245+0000","last_unstale":"2026-02-22T22:37:36.547113+0000","last_undegraded":"2026-02-22T22:37:36.547113+0000","last_fullsized":"2026-02-22T22:37:36.547113+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:06:29.211864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068412599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.545375+0000","last_change":"2026-02-22T22:37:36.545375+0000","last_active":"2026-02-22T22:37:36.545375+0000","last_peered":"2026-02-22T22:37:36.545375+0000","last_clean":"2026-02-22T22:37:36.545375+0000","last_became_active":"2026-02-22T22:37:34.543078+0000","last_became_peered":"2026-02-22T22:37:34.543078+0000","last_unstale":"2026-02-22T22:37:36.545375+0000","last_undegraded":"2026-02-22T22:37:36.545375+0000","last_fullsized":"2026-02-22T22:37:36.545375+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:44:07.853738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066853699999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.542745+0000","last_change":"2026-02-22T22:37:36.542745+0000","last_active":"2026-02-22T22:37:36.542745+0000","last_peered":"2026-02-22T22:37:36.542745+0000","last_clean":"2026-02-22T22:37:36.542745+0000","last_became_active":"2026-02-22T22:37:34.542908+0000","last_became_peered":"2026-02-22T22:37:34.542908+0000","last_unstale":"2026-02-22T22:37:36.542745+0000","last_undegraded":"2026-02-22T22:37:36.542745+0000","last_fullsized":"2026-02-22T22:37:36.542745+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:47:13.400204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043051299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.542742+0000","last_change":"2026-02-22T22:37:36.542742+0000","last_active":"2026-02-22T22:37:36.542742+0000","last_peered":"2026-02-22T22:37:36.542742+0000","last_clean":"2026-02-22T22:37:36.542742+0000","last_became_active":"2026-02-22T22:37:34.542961+0000","last_became_peered":"2026-02-22T22:37:34.542961+0000","last_unstale":"2026-02-22T22:37:36.542742+0000","last_undegraded":"2026-02-22T22:37:36.542742+0000","last_fullsized":"2026-02-22T22:37:36.542742+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:14.729623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000344994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.546970+0000","last_change":"2026-02-22T22:37:36.546970+0000","last_active":"2026-02-22T22:37:36.546970+0000","last_peered":"2026-02-22T22:37:36.546970+0000","last_clean":"2026-02-22T22:37:36.546970+0000","last_became_active":"2026-02-22T22:37:34.542584+0000","last_became_peered":"2026-02-22T22:37:34.542584+0000","last_unstale":"2026-02-22T22:37:36.546970+0000","last_undegraded":"2026-02-22T22:37:36.546970+0000","last_fullsized":"2026-02-22T22:37:36.546970+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:43:40.952490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060424800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:37:36.546580+0000","last_change":"2026-02-22T22:37:30.530613+0000","last_active":"2026-02-22T22:37:36.546580+0000","last_peered":"2026-02-22T22:37:36.546580+0000","last_clean":"2026-02-22T22:37:36.546580+0000","last_became_active":"2026-02-22T22:37:30.530398+0000","last_became_peered":"2026-02-22T22:37:30.530398+0000","last_unstale":"2026-02-22T22:37:36.546580+0000","last_undegraded":"2026-02-22T22:37:36.546580+0000","last_fullsized":"2026-02-22T22:37:36.546580+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_clean_scrub_stamp":"2026-02-22T22:37:29.526086+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:41:58.671728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":0,"up_from":9,"seq":38654705669,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27688,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344152,"statfs":{"total":96636764160,"available":96608411648,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705669,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27104,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344736,"statfs":{"total":96636764160,"available":96609009664,"internally_reserved":0,"allocated":139264,"data_stored":40715,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27589064},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738372,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":631102,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:37:40.149 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:37:40.150 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-22T22:37:40.150 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-22T22:37:40.150 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-22T22:37:40.322 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:40.323 INFO:teuthology.orchestra.run.trial078.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T22:37:40.331 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-22T22:37:40.331 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T22:37:40.374 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-22T22:37:40.374 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T22:37:40.374 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-bootstrap-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64259', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror_bootstrap.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'bluestore': True, 'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs'}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial078.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBDNDWZiJW4+mOO5x9eBXVo1SH1y+4ZVaepjXMZrz5ZUr76fZJVBweiNldE6oG8JquHGeOG7ACQ7DVVHDO5xMleY=', 'trial178.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBMgjfWFN++E60nfU1X/ZHTxvvvI+OpzCzJpV7wLGORR/Gntf4Ene1CfaJ7Lliu5GSNruuGy7XXe+1TU5E4tsGbg='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_bootstrap.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'image', 'RBD_MIRROR_INSTANCES': '1', 'RBD_MIRROR_MODE': 'snapshot', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T22:37:40.375 DEBUG:teuthology.orchestra.run.trial078:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T22:37:40.378 DEBUG:teuthology.orchestra.run.trial178:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T22:37:40.396 DEBUG:teuthology.orchestra.run.trial078:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:37:40.434 DEBUG:teuthology.orchestra.run.trial178:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:37:40.461 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:40.461 DEBUG:teuthology.orchestra.run.trial178:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T22:37:40.517 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T22:37:40.517 DEBUG:teuthology.orchestra.run.trial178:> stat /dev/vg_nvme/lv_1 2026-02-22T22:37:40.574 INFO:teuthology.orchestra.run.trial178.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T22:37:40.575 INFO:teuthology.orchestra.run.trial178.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:37:40.575 INFO:teuthology.orchestra.run.trial178.stdout:Device: 6h/6d Inode: 1310 Links: 1 2026-02-22T22:37:40.575 INFO:teuthology.orchestra.run.trial178.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:37:40.575 INFO:teuthology.orchestra.run.trial178.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:37:40.575 INFO:teuthology.orchestra.run.trial178.stdout:Access: 2026-02-22 22:36:55.352858264 +0000 2026-02-22T22:37:40.575 INFO:teuthology.orchestra.run.trial178.stdout:Modify: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.575 INFO:teuthology.orchestra.run.trial178.stdout:Change: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.575 INFO:teuthology.orchestra.run.trial178.stdout: Birth: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.576 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T22:37:40.638 INFO:teuthology.orchestra.run.trial178.stderr:1+0 records in 2026-02-22T22:37:40.638 INFO:teuthology.orchestra.run.trial178.stderr:1+0 records out 2026-02-22T22:37:40.638 INFO:teuthology.orchestra.run.trial178.stderr:512 bytes copied, 0.000144364 s, 3.5 MB/s 2026-02-22T22:37:40.640 DEBUG:teuthology.orchestra.run.trial178:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T22:37:40.695 DEBUG:teuthology.orchestra.run.trial178:> stat /dev/vg_nvme/lv_2 2026-02-22T22:37:40.752 INFO:teuthology.orchestra.run.trial178.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T22:37:40.752 INFO:teuthology.orchestra.run.trial178.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:37:40.752 INFO:teuthology.orchestra.run.trial178.stdout:Device: 6h/6d Inode: 1316 Links: 1 2026-02-22T22:37:40.752 INFO:teuthology.orchestra.run.trial178.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:37:40.752 INFO:teuthology.orchestra.run.trial178.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:37:40.752 INFO:teuthology.orchestra.run.trial178.stdout:Access: 2026-02-22 22:36:55.352858264 +0000 2026-02-22T22:37:40.753 INFO:teuthology.orchestra.run.trial178.stdout:Modify: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.753 INFO:teuthology.orchestra.run.trial178.stdout:Change: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.753 INFO:teuthology.orchestra.run.trial178.stdout: Birth: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.753 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T22:37:40.814 INFO:teuthology.orchestra.run.trial178.stderr:1+0 records in 2026-02-22T22:37:40.814 INFO:teuthology.orchestra.run.trial178.stderr:1+0 records out 2026-02-22T22:37:40.814 INFO:teuthology.orchestra.run.trial178.stderr:512 bytes copied, 0.000140808 s, 3.6 MB/s 2026-02-22T22:37:40.816 DEBUG:teuthology.orchestra.run.trial178:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T22:37:40.876 DEBUG:teuthology.orchestra.run.trial178:> stat /dev/vg_nvme/lv_3 2026-02-22T22:37:40.934 INFO:teuthology.orchestra.run.trial178.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T22:37:40.934 INFO:teuthology.orchestra.run.trial178.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:37:40.934 INFO:teuthology.orchestra.run.trial178.stdout:Device: 6h/6d Inode: 1312 Links: 1 2026-02-22T22:37:40.934 INFO:teuthology.orchestra.run.trial178.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:37:40.934 INFO:teuthology.orchestra.run.trial178.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:37:40.934 INFO:teuthology.orchestra.run.trial178.stdout:Access: 2026-02-22 22:36:55.353858229 +0000 2026-02-22T22:37:40.934 INFO:teuthology.orchestra.run.trial178.stdout:Modify: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.935 INFO:teuthology.orchestra.run.trial178.stdout:Change: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.935 INFO:teuthology.orchestra.run.trial178.stdout: Birth: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:40.935 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T22:37:40.996 INFO:teuthology.orchestra.run.trial178.stderr:1+0 records in 2026-02-22T22:37:40.996 INFO:teuthology.orchestra.run.trial178.stderr:1+0 records out 2026-02-22T22:37:40.996 INFO:teuthology.orchestra.run.trial178.stderr:512 bytes copied, 0.000149294 s, 3.4 MB/s 2026-02-22T22:37:40.997 DEBUG:teuthology.orchestra.run.trial178:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T22:37:41.054 DEBUG:teuthology.orchestra.run.trial178:> stat /dev/vg_nvme/lv_4 2026-02-22T22:37:41.111 INFO:teuthology.orchestra.run.trial178.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T22:37:41.111 INFO:teuthology.orchestra.run.trial178.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:37:41.111 INFO:teuthology.orchestra.run.trial178.stdout:Device: 6h/6d Inode: 1304 Links: 1 2026-02-22T22:37:41.111 INFO:teuthology.orchestra.run.trial178.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:37:41.111 INFO:teuthology.orchestra.run.trial178.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:37:41.111 INFO:teuthology.orchestra.run.trial178.stdout:Access: 2026-02-22 22:36:55.353858229 +0000 2026-02-22T22:37:41.111 INFO:teuthology.orchestra.run.trial178.stdout:Modify: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:41.112 INFO:teuthology.orchestra.run.trial178.stdout:Change: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:41.112 INFO:teuthology.orchestra.run.trial178.stdout: Birth: 2026-02-22 22:36:21.275076304 +0000 2026-02-22T22:37:41.112 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T22:37:41.173 INFO:teuthology.orchestra.run.trial178.stderr:1+0 records in 2026-02-22T22:37:41.173 INFO:teuthology.orchestra.run.trial178.stderr:1+0 records out 2026-02-22T22:37:41.173 INFO:teuthology.orchestra.run.trial178.stderr:512 bytes copied, 0.000143843 s, 3.6 MB/s 2026-02-22T22:37:41.174 DEBUG:teuthology.orchestra.run.trial178:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T22:37:41.230 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:37:41.230 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial178.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:37:41.231 INFO:tasks.ceph:Generating config... 2026-02-22T22:37:41.233 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T22:37:41.233 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T22:37:41.233 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.234 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.235 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.235 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.235 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.235 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.235 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T22:37:41.235 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T22:37:41.235 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T22:37:41.235 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T22:37:41.235 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T22:37:41.235 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T22:37:41.235 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T22:37:41.236 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] bluestore allocator = stupid 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T22:37:41.236 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T22:37:41.237 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T22:37:41.237 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T22:37:41.237 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T22:37:41.237 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T22:37:41.237 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-22T22:37:41.237 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-22T22:37:41.304 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.keyring 2026-02-22T22:37:41.306 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-22T22:37:41.382 DEBUG:teuthology.orchestra.run.trial178:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:37:41.453 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.178')] 2026-02-22T22:37:41.454 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.178', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore allocator': 'stupid', 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T22:37:41.455 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:41.455 DEBUG:teuthology.orchestra.run.trial178:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:37:41.511 DEBUG:teuthology.orchestra.run.trial178:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.178 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:monmaptool: generated fsid 24cee112-44da-4e56-845c-136a54a559f9 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:setting min_mon_release = quincy 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:epoch 0 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:fsid 24cee112-44da-4e56-845c-136a54a559f9 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:last_changed 2026-02-22T22:37:41.582200+0000 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:created 2026-02-22T22:37:41.582200+0000 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:min_mon_release 17 (quincy) 2026-02-22T22:37:41.585 INFO:teuthology.orchestra.run.trial178.stdout:election_strategy: 1 2026-02-22T22:37:41.586 INFO:teuthology.orchestra.run.trial178.stdout:0: [v2:10.20.193.178:3300/0,v1:10.20.193.178:6789/0] mon.a 2026-02-22T22:37:41.586 INFO:teuthology.orchestra.run.trial178.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-22T22:37:41.587 DEBUG:teuthology.orchestra.run.trial178:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:37:41.643 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 24cee112-44da-4e56-845c-136a54a559f9... 2026-02-22T22:37:41.644 DEBUG:teuthology.orchestra.run.trial078:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T22:37:41.647 DEBUG:teuthology.orchestra.run.trial178:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T22:37:41.686 INFO:teuthology.orchestra.run.trial078.stdout:[global] 2026-02-22T22:37:41.686 INFO:teuthology.orchestra.run.trial078.stdout: chdir = "" 2026-02-22T22:37:41.686 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:37:41.686 INFO:teuthology.orchestra.run.trial078.stdout: auth supported = cephx 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: filestore xattr use omap = true 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: mon clock drift allowed = 1.000 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: osd crush chooseleaf type = 0 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: auth debug = true 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: ms die on old message = true 2026-02-22T22:37:41.687 INFO:teuthology.orchestra.run.trial078.stdout: ms die on bug = true 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: mon pg warn max object skew = 0 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: osd pool default size = 2 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: mon osd allow primary affinity = true 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: mon osd allow pg remap = true 2026-02-22T22:37:41.688 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on no sortbitwise = false 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on too few osds = false 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: mon_allow_pool_size_one = true 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: osd default data pool replay window = 5 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.689 INFO:teuthology.orchestra.run.trial078.stdout: mon allow pool delete = true 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: mon cluster log file level = debug 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: debug asserts on shutdown = true 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: mon health detail to clog = false 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: mon host = 10.20.193.178 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: mon client directed command retry = 5 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: mon warn on pool no app = False 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: ms inject socket failures = 5000 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: fsid = 24cee112-44da-4e56-845c-136a54a559f9 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout:[osd] 2026-02-22T22:37:41.690 INFO:teuthology.orchestra.run.trial078.stdout: osd journal size = 100 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd scrub load threshold = 5.0 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd scrub max interval = 600 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd mclock skip benchmark = true 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd recover clone overlap = true 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd recovery max chunk = 1048576 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd debug shutdown = true 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd debug op order = true 2026-02-22T22:37:41.691 INFO:teuthology.orchestra.run.trial078.stdout: osd debug verify stray on activate = true 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: osd debug trim objects = true 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: osd open classes on start = true 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: osd debug pg log writeout = true 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: osd map max advance = 10 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: journal zero on create = true 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:37:41.692 INFO:teuthology.orchestra.run.trial078.stdout: filestore apply finisher threads = 3 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: bdev debug aio = true 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: osd debug misdirected ops = true 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: bdev async discard = True 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: bdev enable discard = True 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: bluestore allocator = stupid 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: bluestore block size = 96636764160 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: bluestore fsck on mount = True 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: debug bluefs = 1/20 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: debug bluestore = 1/20 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: debug ms = 1 2026-02-22T22:37:41.693 INFO:teuthology.orchestra.run.trial078.stdout: debug osd = 20 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: debug rocksdb = 4/10 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: mon osd full ratio = 0.9 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: osd objectstore = bluestore 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout:[mgr] 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: debug ms = 1 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: debug mgr = 20 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: debug mon = 20 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: debug auth = 20 2026-02-22T22:37:41.694 INFO:teuthology.orchestra.run.trial078.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: mgr/telemetry/nag = false 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout:[mon] 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: debug ms = 1 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: debug mon = 20 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: debug paxos = 20 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: debug auth = 20 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: mon data avail warn = 5 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: mon mgr mkfs grace = 240 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: mon osd reporter subtree level = osd 2026-02-22T22:37:41.695 INFO:teuthology.orchestra.run.trial078.stdout: mon osd prime pg temp = true 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: # 1m isn't quite enough 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:37:41.696 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout:[client] 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: rgw cache enabled = true 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: rgw enable ops log = true 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: rgw enable usage log = true 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: debug rbd = 20 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: debug rbd_mirror = 15 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: log to stderr = False 2026-02-22T22:37:41.697 INFO:teuthology.orchestra.run.trial078.stdout: rbd default features = 125 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout:[mon.a] 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.0] 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.1] 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.2] 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.3] 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.698 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.4] 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.5] 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout:[client.mirror.6] 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.699 INFO:teuthology.orchestra.run.trial078.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout:[global] 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: chdir = "" 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: auth supported = cephx 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: filestore xattr use omap = true 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: mon clock drift allowed = 1.000 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.722 INFO:teuthology.orchestra.run.trial178.stdout: osd crush chooseleaf type = 0 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: auth debug = true 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: ms die on old message = true 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: ms die on bug = true 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: mon pg warn max object skew = 0 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.723 INFO:teuthology.orchestra.run.trial178.stdout: osd pool default size = 2 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon osd allow primary affinity = true 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon osd allow pg remap = true 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on no sortbitwise = false 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on too few osds = false 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: mon_allow_pool_size_one = true 2026-02-22T22:37:41.724 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: osd default data pool replay window = 5 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: mon allow pool delete = true 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: mon cluster log file level = debug 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: debug asserts on shutdown = true 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: mon health detail to clog = false 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: mon host = 10.20.193.178 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: mon client directed command retry = 5 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: mon warn on pool no app = False 2026-02-22T22:37:41.725 INFO:teuthology.orchestra.run.trial178.stdout: ms inject socket failures = 5000 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: fsid = 24cee112-44da-4e56-845c-136a54a559f9 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout:[osd] 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: osd journal size = 100 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: osd scrub load threshold = 5.0 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: osd scrub max interval = 600 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: osd mclock skip benchmark = true 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: osd recover clone overlap = true 2026-02-22T22:37:41.726 INFO:teuthology.orchestra.run.trial178.stdout: osd recovery max chunk = 1048576 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: osd debug shutdown = true 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: osd debug op order = true 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: osd debug verify stray on activate = true 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: osd debug trim objects = true 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: osd open classes on start = true 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: osd debug pg log writeout = true 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: osd map max advance = 10 2026-02-22T22:37:41.727 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: journal zero on create = true 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: filestore apply finisher threads = 3 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: bdev debug aio = true 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: osd debug misdirected ops = true 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: bdev async discard = True 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: bdev enable discard = True 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: bluestore allocator = stupid 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: bluestore block size = 96636764160 2026-02-22T22:37:41.728 INFO:teuthology.orchestra.run.trial178.stdout: bluestore fsck on mount = True 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: debug bluefs = 1/20 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: debug bluestore = 1/20 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: debug ms = 1 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: debug osd = 20 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: debug rocksdb = 4/10 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: mon osd full ratio = 0.9 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: osd objectstore = bluestore 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout:[mgr] 2026-02-22T22:37:41.729 INFO:teuthology.orchestra.run.trial178.stdout: debug ms = 1 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: debug mgr = 20 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: debug mon = 20 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: debug auth = 20 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: mgr/telemetry/nag = false 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout:[mon] 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: debug ms = 1 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: debug mon = 20 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: debug paxos = 20 2026-02-22T22:37:41.730 INFO:teuthology.orchestra.run.trial178.stdout: debug auth = 20 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: mon data avail warn = 5 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: mon mgr mkfs grace = 240 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: mon osd reporter subtree level = osd 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: mon osd prime pg temp = true 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:37:41.731 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: # 1m isn't quite enough 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout:[client] 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: rgw cache enabled = true 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: rgw enable ops log = true 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: rgw enable usage log = true 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:37:41.732 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: debug rbd = 20 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: debug rbd_mirror = 15 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: log to stderr = False 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: rbd default features = 125 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout:[mon.a] 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.0] 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.1] 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.2] 2026-02-22T22:37:41.733 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.3] 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.4] 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.5] 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.6] 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:37:41.734 INFO:teuthology.orchestra.run.trial178.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:37:41.736 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-22T22:37:41.736 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-22T22:37:41.817 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T22:37:41.817 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:41.818 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-22T22:37:41.876 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:41.876 DEBUG:teuthology.orchestra.run.trial178:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-22T22:37:41.932 INFO:tasks.ceph:Sending monmap to node ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:37:41.932 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:41.932 DEBUG:teuthology.orchestra.run.trial078:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T22:37:41.933 DEBUG:teuthology.orchestra.run.trial078:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:37:41.968 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:41.968 DEBUG:teuthology.orchestra.run.trial078:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:37:42.024 INFO:tasks.ceph:Sending monmap to node ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:37:42.024 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:42.024 DEBUG:teuthology.orchestra.run.trial178:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T22:37:42.024 DEBUG:teuthology.orchestra.run.trial178:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:37:42.059 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:42.059 DEBUG:teuthology.orchestra.run.trial178:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:37:42.115 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T22:37:42.116 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T22:37:42.116 DEBUG:teuthology.orchestra.run.trial078:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T22:37:42.162 INFO:teuthology.orchestra.run.trial078.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T22:37:42.164 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T22:37:42.165 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T22:37:42.165 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-22T22:37:42.199 INFO:teuthology.orchestra.run.trial078.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-22T22:37:42.210 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T22:37:42.245 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T22:37:42.256 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T22:37:42.339 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T22:37:42.351 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T22:37:42.423 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T22:37:42.434 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T22:37:42.509 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T22:37:42.520 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T22:37:42.593 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T22:37:42.604 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T22:37:42.678 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T22:37:42.689 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T22:37:42.762 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T22:37:42.773 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T22:37:42.846 INFO:teuthology.orchestra.run.trial178.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T22:37:42.857 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T22:37:42.857 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial078.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial178.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:37:42.858 DEBUG:teuthology.orchestra.run.trial178:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-22T22:37:42.923 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:37:42.924 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-22T22:37:42.924 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:37:42.924 DEBUG:teuthology.orchestra.run.trial178:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T22:37:42.988 INFO:teuthology.orchestra.run.trial178.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:37:42.988 INFO:teuthology.orchestra.run.trial178.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:37:42.988 INFO:teuthology.orchestra.run.trial178.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:37:42.988 INFO:teuthology.orchestra.run.trial178.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:37:42.988 INFO:teuthology.orchestra.run.trial178.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:37:42.989 INFO:teuthology.orchestra.run.trial178.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:37:42.989 INFO:teuthology.orchestra.run.trial178.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:37:42.989 INFO:teuthology.orchestra.run.trial178.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:37:42.989 INFO:teuthology.orchestra.run.trial178.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:37:42.989 INFO:teuthology.orchestra.run.trial178.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:37:43.549 INFO:teuthology.orchestra.run.trial178.stdout:Discarding blocks...Done. 2026-02-22T22:37:43.552 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial178.front.sepia.ceph.com -o noatime 2026-02-22T22:37:43.552 DEBUG:teuthology.orchestra.run.trial178:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-22T22:37:43.625 DEBUG:teuthology.orchestra.run.trial178:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-22T22:37:43.693 DEBUG:teuthology.orchestra.run.trial178:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-22T22:37:43.756 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:37:43.756 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-22T22:37:43.756 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:37:43.757 DEBUG:teuthology.orchestra.run.trial178:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T22:37:43.823 INFO:teuthology.orchestra.run.trial178.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:37:43.823 INFO:teuthology.orchestra.run.trial178.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:37:43.823 INFO:teuthology.orchestra.run.trial178.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:37:43.823 INFO:teuthology.orchestra.run.trial178.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:37:43.824 INFO:teuthology.orchestra.run.trial178.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:37:43.824 INFO:teuthology.orchestra.run.trial178.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:37:43.824 INFO:teuthology.orchestra.run.trial178.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:37:43.824 INFO:teuthology.orchestra.run.trial178.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:37:43.824 INFO:teuthology.orchestra.run.trial178.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:37:43.824 INFO:teuthology.orchestra.run.trial178.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:37:44.391 INFO:teuthology.orchestra.run.trial178.stdout:Discarding blocks...Done. 2026-02-22T22:37:44.393 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial178.front.sepia.ceph.com -o noatime 2026-02-22T22:37:44.393 DEBUG:teuthology.orchestra.run.trial178:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-22T22:37:44.459 DEBUG:teuthology.orchestra.run.trial178:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-22T22:37:44.526 DEBUG:teuthology.orchestra.run.trial178:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-22T22:37:44.590 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:37:44.590 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-22T22:37:44.591 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:37:44.591 DEBUG:teuthology.orchestra.run.trial178:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T22:37:44.657 INFO:teuthology.orchestra.run.trial178.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:37:44.657 INFO:teuthology.orchestra.run.trial178.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:37:44.657 INFO:teuthology.orchestra.run.trial178.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:37:44.657 INFO:teuthology.orchestra.run.trial178.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:37:44.657 INFO:teuthology.orchestra.run.trial178.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:37:44.658 INFO:teuthology.orchestra.run.trial178.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:37:44.658 INFO:teuthology.orchestra.run.trial178.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:37:44.658 INFO:teuthology.orchestra.run.trial178.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:37:44.658 INFO:teuthology.orchestra.run.trial178.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:37:44.658 INFO:teuthology.orchestra.run.trial178.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:37:45.217 INFO:teuthology.orchestra.run.trial178.stdout:Discarding blocks...Done. 2026-02-22T22:37:45.220 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial178.front.sepia.ceph.com -o noatime 2026-02-22T22:37:45.220 DEBUG:teuthology.orchestra.run.trial178:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-22T22:37:45.288 DEBUG:teuthology.orchestra.run.trial178:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-22T22:37:45.356 DEBUG:teuthology.orchestra.run.trial178:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:37:45.434 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:45.430+0000 7f670b516780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-22T22:37:45.435 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:45.431+0000 7f670b516780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-22T22:37:45.435 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:45.431+0000 7f670b516780 -1 bdev(0x557631c49800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-22T22:37:45.435 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:45.431+0000 7f670b516780 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-22T22:37:49.329 DEBUG:teuthology.orchestra.run.trial178:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-22T22:37:49.394 DEBUG:teuthology.orchestra.run.trial178:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:37:49.472 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:49.468+0000 7f6234c82780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-22T22:37:49.472 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:49.468+0000 7f6234c82780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-22T22:37:49.472 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:49.468+0000 7f6234c82780 -1 bdev(0x55bf73033800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-22T22:37:49.472 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:49.468+0000 7f6234c82780 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-22T22:37:53.368 DEBUG:teuthology.orchestra.run.trial178:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-22T22:37:53.434 DEBUG:teuthology.orchestra.run.trial178:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:37:53.511 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:53.507+0000 7fcea78fe780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-22T22:37:53.511 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:53.508+0000 7fcea78fe780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-22T22:37:53.511 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:53.508+0000 7fcea78fe780 -1 bdev(0x555e6938f800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-22T22:37:53.511 INFO:teuthology.orchestra.run.trial178.stderr:2026-02-22T22:37:53.508+0000 7fcea78fe780 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-22T22:37:57.403 DEBUG:teuthology.orchestra.run.trial178:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-22T22:37:57.469 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T22:37:57.470 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:57.470 DEBUG:teuthology.orchestra.run.trial078:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-22T22:37:57.497 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.497 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-22T22:37:57.532 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.533 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-22T22:37:57.598 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.598 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-22T22:37:57.662 DEBUG:teuthology.orchestra.run.trial078:> set -ex 2026-02-22T22:37:57.662 DEBUG:teuthology.orchestra.run.trial078:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-22T22:37:57.681 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.681 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-22T22:37:57.717 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.718 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-22T22:37:57.774 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.774 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-22T22:37:57.830 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.830 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-22T22:37:57.886 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.886 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-22T22:37:57.942 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:57.942 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-22T22:37:58.000 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:58.001 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-22T22:37:58.057 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:58.057 DEBUG:teuthology.orchestra.run.trial178:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-22T22:37:58.113 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T22:37:58.113 DEBUG:teuthology.orchestra.run.trial178:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout:[mgr.x] 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtppY6NCRAASv2AvnXNs/JI53FDBIzU/A== 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout:[osd.0] 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA5hZtpnHG7GRAAXrE8YlIQf7k8OlVFlvijrA== 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout:[osd.1] 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA9hZtpP0b6GxAAe0Y9rTkFwx6/ky3Qlu25UQ== 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout:[osd.2] 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout: key = AQBBhZtpa69VHhAAVDQVIVsrNCWzvx9648U1TQ== 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout:[client.0] 2026-02-22T22:37:58.175 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpVSS/CxAAzeOIBPbjKZ/xPCWIgnVkpA== 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror] 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtppl19DhAAeg0XJcUi0i2h/XEsl7RzEw== 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.0] 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpWb8YFBAAdBXYRn6iqO6TICYIN4wBKw== 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.1] 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpg4UaGRAAvSm1zYDrzFMEkJQKeCcQ+g== 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.2] 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpz2kuHhAAYMwTRzJb5CeUCQLpq2R3iQ== 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.3] 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtplBU0IxAAyT4ImepXFH6xsnLghjdKUw== 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.4] 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpRHZHKBAArGUqNt76mmUtH6pyUVXYXA== 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.5] 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpQyxGLRAARJYkVzbX+KZeONXSkgXMvQ== 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.6] 2026-02-22T22:37:58.176 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpGmlMMhAAr10b8YosCw0J9S+hc08ZUQ== 2026-02-22T22:37:58.177 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T22:37:58.258 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:37:58.338 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:37:58.418 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:37:58.497 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:58.579 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:58.658 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:58.739 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:58.820 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:58.899 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:58.979 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:59.058 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:59.138 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:37:59.217 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T22:37:59.218 DEBUG:teuthology.orchestra.run.trial178:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-22T22:37:59.282 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-22T22:37:59.368 DEBUG:teuthology.orchestra.run.trial178:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-22T22:37:59.393 DEBUG:teuthology.orchestra.run.trial178:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:37:59.450 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-22T22:37:59.450 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-22T22:37:59.450 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-22T22:37:59.492 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-22T22:37:59.492 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-22T22:37:59.492 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-22T22:37:59.493 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-22T22:37:59.495 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-22T22:37:59.496 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T22:37:59.496 DEBUG:teuthology.orchestra.run.trial178:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-22T22:37:59.601 INFO:teuthology.orchestra.run.trial178.stderr:adjusted tunables profile to default 2026-02-22T22:37:59.609 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T22:37:59.610 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-22T22:37:59.610 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:59.610 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-22T22:37:59.634 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:59.635 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-22T22:37:59.699 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:37:59.700 DEBUG:teuthology.orchestra.run.trial178:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-22T22:37:59.763 DEBUG:teuthology.orchestra.run.trial178:> sudo ceph --cluster cluster2 osd new f2b02fcf-c25e-422e-bed4-e1b897f6ec53 0 2026-02-22T22:37:59.833 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:37:59.830+0000 7f5c45ba5140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T22:37:59.911 INFO:teuthology.orchestra.run.trial178.stdout:0 2026-02-22T22:37:59.920 DEBUG:teuthology.orchestra.run.trial178:> sudo ceph --cluster cluster2 osd new b2a9d9d6-5251-46f8-8eea-9cb5b8a7d7fb 1 2026-02-22T22:37:59.949 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:37:59.946+0000 7f5c45ba5140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T22:37:59.971 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:37:59.968+0000 7f5c45ba5140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T22:37:59.996 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:37:59.993+0000 7f5c45ba5140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T22:38:00.017 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.014+0000 7f5c45ba5140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T22:38:00.023 INFO:teuthology.orchestra.run.trial178.stdout:1 2026-02-22T22:38:00.032 DEBUG:teuthology.orchestra.run.trial178:> sudo ceph --cluster cluster2 osd new f28d8f58-67be-4316-bb40-0fad368ef7fe 2 2026-02-22T22:38:00.042 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.039+0000 7f5c45ba5140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T22:38:00.093 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.090+0000 7f5c45ba5140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T22:38:00.135 INFO:teuthology.orchestra.run.trial178.stdout:2 2026-02-22T22:38:00.142 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-22T22:38:00.143 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-22T22:38:00.145 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-22T22:38:00.145 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-22T22:38:00.145 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-22T22:38:00.148 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-22T22:38:00.148 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-22T22:38:00.148 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-22T22:38:00.150 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-22T22:38:00.151 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:38:00.152 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T22:38:00.153 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T22:38:00.153 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr: from numpy import show_config as show_numpy_config 2026-02-22T22:38:00.155 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.152+0000 7f5c45ba5140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T22:38:00.179 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.176+0000 7f5c45ba5140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T22:38:00.233 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.230+0000 7f5c45ba5140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T22:38:00.251 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:38:00.252 INFO:teuthology.orchestra.run.trial178.stdout:{"epoch":5,"fsid":"24cee112-44da-4e56-845c-136a54a559f9","created":"2026-02-22T22:37:59.549109+0000","modified":"2026-02-22T22:38:00.132360+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:38:00.132360+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f2b02fcf-c25e-422e-bed4-e1b897f6ec53","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b2a9d9d6-5251-46f8-8eea-9cb5b8a7d7fb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f28d8f58-67be-4316-bb40-0fad368ef7fe","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:38:00.259 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-22T22:38:00.260 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T22:38:00.274 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.271+0000 7f5c45ba5140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T22:38:00.298 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.295+0000 7f5c45ba5140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T22:38:00.321 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.318+0000 7f5c45ba5140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T22:38:00.345 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.342+0000 7f5c45ba5140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T22:38:00.366 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.363+0000 7f5c45ba5140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T22:38:00.520 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.517+0000 7f5c45ba5140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T22:38:00.562 DEBUG:teuthology.orchestra.run.trial178:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:38:00.576 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.573+0000 7f5c45ba5140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T22:38:00.597 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.594+0000 7f5c45ba5140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T22:38:00.619 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.616+0000 7f5c45ba5140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T22:38:00.655 INFO:teuthology.misc.health.trial178.stdout: 2026-02-22T22:38:00.655 INFO:teuthology.misc.health.trial178.stdout:{"epoch":5,"fsid":"24cee112-44da-4e56-845c-136a54a559f9","created":"2026-02-22T22:37:59.549109+0000","modified":"2026-02-22T22:38:00.132360+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:38:00.132360+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f2b02fcf-c25e-422e-bed4-e1b897f6ec53","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b2a9d9d6-5251-46f8-8eea-9cb5b8a7d7fb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f28d8f58-67be-4316-bb40-0fad368ef7fe","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:38:00.661 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:38:00.671 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.669+0000 7f5c45ba5140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:38:00.692 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.689+0000 7f5c45ba5140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T22:38:00.744 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:00.742+0000 7f5c45ba5140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:38:01.066 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:01.063+0000 7f5c45ba5140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T22:38:01.087 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:01.085+0000 7f5c45ba5140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T22:38:01.421 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:01.418+0000 7f5c45ba5140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T22:38:02.041 INFO:tasks.ceph.cluster2.osd.0.trial178.stderr:2026-02-22T22:38:02.037+0000 7fe588bd4780 -1 Falling back to public interface 2026-02-22T22:38:02.062 INFO:tasks.ceph.cluster2.osd.1.trial178.stderr:2026-02-22T22:38:02.058+0000 7f4807719780 -1 Falling back to public interface 2026-02-22T22:38:02.078 INFO:tasks.ceph.cluster2.osd.2.trial178.stderr:2026-02-22T22:38:02.074+0000 7f1f6d319780 -1 Falling back to public interface 2026-02-22T22:38:02.428 INFO:tasks.ceph.cluster2.mgr.x.trial078.stderr:2026-02-22T22:38:02.425+0000 7f5b632e6640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T22:38:06.501 INFO:tasks.ceph.cluster2.osd.0.trial178.stderr:2026-02-22T22:38:06.497+0000 7fe588bd4780 -1 osd.0 0 log_to_monitors true 2026-02-22T22:38:06.543 INFO:tasks.ceph.cluster2.osd.1.trial178.stderr:2026-02-22T22:38:06.539+0000 7f4807719780 -1 osd.1 0 log_to_monitors true 2026-02-22T22:38:06.577 INFO:tasks.ceph.cluster2.osd.2.trial178.stderr:2026-02-22T22:38:06.573+0000 7f1f6d319780 -1 osd.2 0 log_to_monitors true 2026-02-22T22:38:06.964 DEBUG:teuthology.orchestra.run.trial178:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:38:07.118 INFO:teuthology.misc.health.trial178.stdout: 2026-02-22T22:38:07.118 INFO:teuthology.misc.health.trial178.stdout:{"epoch":6,"fsid":"24cee112-44da-4e56-845c-136a54a559f9","created":"2026-02-22T22:37:59.549109+0000","modified":"2026-02-22T22:38:06.550421+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:38:00.132360+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"f2b02fcf-c25e-422e-bed4-e1b897f6ec53","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"b2a9d9d6-5251-46f8-8eea-9cb5b8a7d7fb","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"f28d8f58-67be-4316-bb40-0fad368ef7fe","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:38:07.125 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:38:07.567 INFO:tasks.ceph.cluster2.osd.1.trial178.stderr:2026-02-22T22:38:07.564+0000 7f4803686640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T22:38:07.568 INFO:tasks.ceph.cluster2.osd.0.trial178.stderr:2026-02-22T22:38:07.564+0000 7fe584b41640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T22:38:07.569 INFO:tasks.ceph.cluster2.osd.0.trial178.stderr:2026-02-22T22:38:07.566+0000 7fe57c974640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:38:07.569 INFO:tasks.ceph.cluster2.osd.1.trial178.stderr:2026-02-22T22:38:07.566+0000 7f47fb4b9640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:38:08.573 INFO:tasks.ceph.cluster2.osd.2.trial178.stderr:2026-02-22T22:38:08.570+0000 7f1f69286640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T22:38:08.575 INFO:tasks.ceph.cluster2.osd.2.trial178.stderr:2026-02-22T22:38:08.572+0000 7f1f610b9640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:38:13.428 DEBUG:teuthology.orchestra.run.trial178:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:38:13.579 INFO:teuthology.misc.health.trial178.stdout: 2026-02-22T22:38:13.579 INFO:teuthology.misc.health.trial178.stdout:{"epoch":11,"fsid":"24cee112-44da-4e56-845c-136a54a559f9","created":"2026-02-22T22:37:59.549109+0000","modified":"2026-02-22T22:38:11.561849+0000","last_up_change":"2026-02-22T22:38:09.558114+0000","last_in_change":"2026-02-22T22:38:00.132360+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:38:09.439442+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f2b02fcf-c25e-422e-bed4-e1b897f6ec53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6800","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6801","nonce":184969568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6802","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6803","nonce":184969568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6806","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6807","nonce":184969568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6804","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6805","nonce":184969568}]},"public_addr":"10.20.193.178:6801/184969568","cluster_addr":"10.20.193.178:6803/184969568","heartbeat_back_addr":"10.20.193.178:6807/184969568","heartbeat_front_addr":"10.20.193.178:6805/184969568","state":["exists","up"]},{"osd":1,"uuid":"b2a9d9d6-5251-46f8-8eea-9cb5b8a7d7fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":9,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6808","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6809","nonce":3461381058}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6810","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6811","nonce":3461381058}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6814","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6815","nonce":3461381058}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6812","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6813","nonce":3461381058}]},"public_addr":"10.20.193.178:6809/3461381058","cluster_addr":"10.20.193.178:6811/3461381058","heartbeat_back_addr":"10.20.193.178:6815/3461381058","heartbeat_front_addr":"10.20.193.178:6813/3461381058","state":["exists","up"]},{"osd":2,"uuid":"f28d8f58-67be-4316-bb40-0fad368ef7fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6816","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6817","nonce":371009371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6818","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6819","nonce":371009371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6822","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6823","nonce":371009371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6820","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6821","nonce":371009371}]},"public_addr":"10.20.193.178:6817/371009371","cluster_addr":"10.20.193.178:6819/371009371","heartbeat_back_addr":"10.20.193.178:6823/371009371","heartbeat_front_addr":"10.20.193.178:6821/371009371","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.496134+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.544189+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.582806+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:38:13.586 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T22:38:13.587 INFO:tasks.ceph:Creating RBD pool 2026-02-22T22:38:13.587 DEBUG:teuthology.orchestra.run.trial178:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-22T22:38:14.571 INFO:teuthology.orchestra.run.trial178.stderr:pool 'rbd' created 2026-02-22T22:38:14.584 DEBUG:teuthology.orchestra.run.trial178:> rbd --cluster cluster2 pool init rbd 2026-02-22T22:38:17.583 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-22T22:38:17.583 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-22T22:38:17.583 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-22T22:38:17.583 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-22T22:38:17.583 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T22:38:17.763 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:38:17.770 INFO:teuthology.orchestra.run.trial178.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6826","nonce":2574387888},{"type":"v1","addr":"10.20.193.78:6827","nonce":2574387888}]},"active_addr":"10.20.193.78:6827/2574387888","active_change":"2026-02-22T22:38:01.420510+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.78:0","nonce":3741361004}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.78:0","nonce":2542496903}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.78:0","nonce":258738475}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.78:0","nonce":3734069822}]}]} 2026-02-22T22:38:17.777 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-22T22:38:17.777 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-22T22:38:17.778 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:38:17.935 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:38:17.935 INFO:teuthology.orchestra.run.trial178.stdout:{"epoch":15,"fsid":"24cee112-44da-4e56-845c-136a54a559f9","created":"2026-02-22T22:37:59.549109+0000","modified":"2026-02-22T22:38:17.572339+0000","last_up_change":"2026-02-22T22:38:09.558114+0000","last_in_change":"2026-02-22T22:38:00.132360+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:38:09.439442+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:38:13.744265+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f2b02fcf-c25e-422e-bed4-e1b897f6ec53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6800","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6801","nonce":184969568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6802","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6803","nonce":184969568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6806","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6807","nonce":184969568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6804","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6805","nonce":184969568}]},"public_addr":"10.20.193.178:6801/184969568","cluster_addr":"10.20.193.178:6803/184969568","heartbeat_back_addr":"10.20.193.178:6807/184969568","heartbeat_front_addr":"10.20.193.178:6805/184969568","state":["exists","up"]},{"osd":1,"uuid":"b2a9d9d6-5251-46f8-8eea-9cb5b8a7d7fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6808","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6809","nonce":3461381058}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6810","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6811","nonce":3461381058}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6814","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6815","nonce":3461381058}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6812","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6813","nonce":3461381058}]},"public_addr":"10.20.193.178:6809/3461381058","cluster_addr":"10.20.193.178:6811/3461381058","heartbeat_back_addr":"10.20.193.178:6815/3461381058","heartbeat_front_addr":"10.20.193.178:6813/3461381058","state":["exists","up"]},{"osd":2,"uuid":"f28d8f58-67be-4316-bb40-0fad368ef7fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6816","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6817","nonce":371009371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6818","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6819","nonce":371009371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6822","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6823","nonce":371009371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6820","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6821","nonce":371009371}]},"public_addr":"10.20.193.178:6817/371009371","cluster_addr":"10.20.193.178:6819/371009371","heartbeat_back_addr":"10.20.193.178:6823/371009371","heartbeat_front_addr":"10.20.193.178:6821/371009371","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.496134+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.544189+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.582806+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:38:17.944 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-22T22:38:17.944 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:38:18.102 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:38:18.102 INFO:teuthology.orchestra.run.trial178.stdout:{"epoch":15,"fsid":"24cee112-44da-4e56-845c-136a54a559f9","created":"2026-02-22T22:37:59.549109+0000","modified":"2026-02-22T22:38:17.572339+0000","last_up_change":"2026-02-22T22:38:09.558114+0000","last_in_change":"2026-02-22T22:38:00.132360+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:38:09.439442+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:38:13.744265+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f2b02fcf-c25e-422e-bed4-e1b897f6ec53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6800","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6801","nonce":184969568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6802","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6803","nonce":184969568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6806","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6807","nonce":184969568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6804","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6805","nonce":184969568}]},"public_addr":"10.20.193.178:6801/184969568","cluster_addr":"10.20.193.178:6803/184969568","heartbeat_back_addr":"10.20.193.178:6807/184969568","heartbeat_front_addr":"10.20.193.178:6805/184969568","state":["exists","up"]},{"osd":1,"uuid":"b2a9d9d6-5251-46f8-8eea-9cb5b8a7d7fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6808","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6809","nonce":3461381058}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6810","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6811","nonce":3461381058}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6814","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6815","nonce":3461381058}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6812","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6813","nonce":3461381058}]},"public_addr":"10.20.193.178:6809/3461381058","cluster_addr":"10.20.193.178:6811/3461381058","heartbeat_back_addr":"10.20.193.178:6815/3461381058","heartbeat_front_addr":"10.20.193.178:6813/3461381058","state":["exists","up"]},{"osd":2,"uuid":"f28d8f58-67be-4316-bb40-0fad368ef7fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":12,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6816","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6817","nonce":371009371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6818","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6819","nonce":371009371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6822","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6823","nonce":371009371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6820","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6821","nonce":371009371}]},"public_addr":"10.20.193.178:6817/371009371","cluster_addr":"10.20.193.178:6819/371009371","heartbeat_back_addr":"10.20.193.178:6823/371009371","heartbeat_front_addr":"10.20.193.178:6821/371009371","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.496134+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.544189+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.582806+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:38:18.111 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-22T22:38:18.112 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-22T22:38:18.112 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-22T22:38:18.200 INFO:teuthology.orchestra.run.trial178.stdout:34359738373 2026-02-22T22:38:18.200 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:38:18.201 INFO:teuthology.orchestra.run.trial178.stdout:38654705668 2026-02-22T22:38:18.201 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:38:18.202 INFO:teuthology.orchestra.run.trial178.stdout:34359738373 2026-02-22T22:38:18.203 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:38:18.399 INFO:teuthology.orchestra.run.trial178.stdout:34359738372 2026-02-22T22:38:18.402 INFO:teuthology.orchestra.run.trial178.stdout:34359738372 2026-02-22T22:38:18.402 INFO:teuthology.orchestra.run.trial178.stdout:38654705667 2026-02-22T22:38:18.408 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.0 2026-02-22T22:38:18.411 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.1 2026-02-22T22:38:18.412 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.2 2026-02-22T22:38:19.409 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:38:19.412 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:38:19.413 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:38:19.569 INFO:teuthology.orchestra.run.trial178.stdout:34359738372 2026-02-22T22:38:19.579 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738372 for osd.0 2026-02-22T22:38:19.612 INFO:teuthology.orchestra.run.trial178.stdout:38654705668 2026-02-22T22:38:19.612 INFO:teuthology.orchestra.run.trial178.stdout:34359738373 2026-02-22T22:38:19.621 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705668 for osd.2 2026-02-22T22:38:19.622 DEBUG:teuthology.parallel:result is None 2026-02-22T22:38:19.622 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.1 2026-02-22T22:38:19.623 DEBUG:teuthology.parallel:result is None 2026-02-22T22:38:20.580 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:38:20.737 INFO:teuthology.orchestra.run.trial178.stdout:34359738373 2026-02-22T22:38:20.746 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738373 got 34359738373 for osd.0 2026-02-22T22:38:20.746 DEBUG:teuthology.parallel:result is None 2026-02-22T22:38:20.746 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:38:20.746 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:38:20.902 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:38:20.902 INFO:teuthology.orchestra.run.trial178.stderr:dumped all 2026-02-22T22:38:20.910 INFO:teuthology.orchestra.run.trial178.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2026-02-22T22:38:19.426531+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82424,"kb_used_data":1520,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033096,"statfs":{"total":289910292480,"available":289825890304,"internally_reserved":0,"allocated":1556480,"data_stored":1282606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.856964"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.577006+0000","last_change":"2026-02-22T22:38:17.577006+0000","last_active":"2026-02-22T22:38:17.577006+0000","last_peered":"2026-02-22T22:38:17.577006+0000","last_clean":"2026-02-22T22:38:17.577006+0000","last_became_active":"2026-02-22T22:38:15.574752+0000","last_became_peered":"2026-02-22T22:38:15.574752+0000","last_unstale":"2026-02-22T22:38:17.577006+0000","last_undegraded":"2026-02-22T22:38:17.577006+0000","last_fullsized":"2026-02-22T22:38:17.577006+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:05:37.350245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082003000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.576762+0000","last_change":"2026-02-22T22:38:17.576762+0000","last_active":"2026-02-22T22:38:17.576762+0000","last_peered":"2026-02-22T22:38:17.576762+0000","last_clean":"2026-02-22T22:38:17.576762+0000","last_became_active":"2026-02-22T22:38:15.574306+0000","last_became_peered":"2026-02-22T22:38:15.574306+0000","last_unstale":"2026-02-22T22:38:17.576762+0000","last_undegraded":"2026-02-22T22:38:17.576762+0000","last_fullsized":"2026-02-22T22:38:17.576762+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:25:28.811738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059383000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.576464+0000","last_change":"2026-02-22T22:38:17.576464+0000","last_active":"2026-02-22T22:38:17.576464+0000","last_peered":"2026-02-22T22:38:17.576464+0000","last_clean":"2026-02-22T22:38:17.576464+0000","last_became_active":"2026-02-22T22:38:15.574673+0000","last_became_peered":"2026-02-22T22:38:15.574673+0000","last_unstale":"2026-02-22T22:38:17.576464+0000","last_undegraded":"2026-02-22T22:38:17.576464+0000","last_fullsized":"2026-02-22T22:38:17.576464+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:46.550560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031834000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.576467+0000","last_change":"2026-02-22T22:38:17.576467+0000","last_active":"2026-02-22T22:38:17.576467+0000","last_peered":"2026-02-22T22:38:17.576467+0000","last_clean":"2026-02-22T22:38:17.576467+0000","last_became_active":"2026-02-22T22:38:15.574763+0000","last_became_peered":"2026-02-22T22:38:15.574763+0000","last_unstale":"2026-02-22T22:38:17.576467+0000","last_undegraded":"2026-02-22T22:38:17.576467+0000","last_fullsized":"2026-02-22T22:38:17.576467+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:25.430048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031511400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.575570+0000","last_change":"2026-02-22T22:38:17.575058+0000","last_active":"2026-02-22T22:38:17.575570+0000","last_peered":"2026-02-22T22:38:17.575570+0000","last_clean":"2026-02-22T22:38:17.575570+0000","last_became_active":"2026-02-22T22:38:15.574409+0000","last_became_peered":"2026-02-22T22:38:15.574409+0000","last_unstale":"2026-02-22T22:38:17.575570+0000","last_undegraded":"2026-02-22T22:38:17.575570+0000","last_fullsized":"2026-02-22T22:38:17.575570+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:32:34.479960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053028299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.677203+0000","last_change":"2026-02-22T22:38:17.677203+0000","last_active":"2026-02-22T22:38:17.677203+0000","last_peered":"2026-02-22T22:38:17.677203+0000","last_clean":"2026-02-22T22:38:17.677203+0000","last_became_active":"2026-02-22T22:38:15.574389+0000","last_became_peered":"2026-02-22T22:38:15.574389+0000","last_unstale":"2026-02-22T22:38:17.677203+0000","last_undegraded":"2026-02-22T22:38:17.677203+0000","last_fullsized":"2026-02-22T22:38:17.677203+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:09:24.494810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021430700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.677186+0000","last_change":"2026-02-22T22:38:17.677186+0000","last_active":"2026-02-22T22:38:17.677186+0000","last_peered":"2026-02-22T22:38:17.677186+0000","last_clean":"2026-02-22T22:38:17.677186+0000","last_became_active":"2026-02-22T22:38:15.574402+0000","last_became_peered":"2026-02-22T22:38:15.574402+0000","last_unstale":"2026-02-22T22:38:17.677186+0000","last_undegraded":"2026-02-22T22:38:17.677186+0000","last_fullsized":"2026-02-22T22:38:17.677186+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:19.472593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019295799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.576831+0000","last_change":"2026-02-22T22:38:17.576831+0000","last_active":"2026-02-22T22:38:17.576831+0000","last_peered":"2026-02-22T22:38:17.576831+0000","last_clean":"2026-02-22T22:38:17.576831+0000","last_became_active":"2026-02-22T22:38:15.574263+0000","last_became_peered":"2026-02-22T22:38:15.574263+0000","last_unstale":"2026-02-22T22:38:17.576831+0000","last_undegraded":"2026-02-22T22:38:17.576831+0000","last_fullsized":"2026-02-22T22:38:17.576831+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:41.940013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068121299999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.575933+0000","last_change":"2026-02-22T22:38:11.502395+0000","last_active":"2026-02-22T22:38:17.575933+0000","last_peered":"2026-02-22T22:38:17.575933+0000","last_clean":"2026-02-22T22:38:17.575933+0000","last_became_active":"2026-02-22T22:38:11.501894+0000","last_became_peered":"2026-02-22T22:38:11.501894+0000","last_unstale":"2026-02-22T22:38:17.575933+0000","last_undegraded":"2026-02-22T22:38:17.575933+0000","last_fullsized":"2026-02-22T22:38:17.575933+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_clean_scrub_stamp":"2026-02-22T22:38:09.558114+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:19:00.206795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":112,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":114688,"data_stored":31634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":720896,"data_stored":625486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":720896,"data_stored":625486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:38:20.911 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:38:21.067 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:38:21.068 INFO:teuthology.orchestra.run.trial178.stderr:dumped all 2026-02-22T22:38:21.076 INFO:teuthology.orchestra.run.trial178.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2026-02-22T22:38:19.426531+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":15,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82424,"kb_used_data":1520,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033096,"statfs":{"total":289910292480,"available":289825890304,"internally_reserved":0,"allocated":1556480,"data_stored":1282606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.856964"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.577006+0000","last_change":"2026-02-22T22:38:17.577006+0000","last_active":"2026-02-22T22:38:17.577006+0000","last_peered":"2026-02-22T22:38:17.577006+0000","last_clean":"2026-02-22T22:38:17.577006+0000","last_became_active":"2026-02-22T22:38:15.574752+0000","last_became_peered":"2026-02-22T22:38:15.574752+0000","last_unstale":"2026-02-22T22:38:17.577006+0000","last_undegraded":"2026-02-22T22:38:17.577006+0000","last_fullsized":"2026-02-22T22:38:17.577006+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:05:37.350245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082003000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.576762+0000","last_change":"2026-02-22T22:38:17.576762+0000","last_active":"2026-02-22T22:38:17.576762+0000","last_peered":"2026-02-22T22:38:17.576762+0000","last_clean":"2026-02-22T22:38:17.576762+0000","last_became_active":"2026-02-22T22:38:15.574306+0000","last_became_peered":"2026-02-22T22:38:15.574306+0000","last_unstale":"2026-02-22T22:38:17.576762+0000","last_undegraded":"2026-02-22T22:38:17.576762+0000","last_fullsized":"2026-02-22T22:38:17.576762+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:25:28.811738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059383000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.576464+0000","last_change":"2026-02-22T22:38:17.576464+0000","last_active":"2026-02-22T22:38:17.576464+0000","last_peered":"2026-02-22T22:38:17.576464+0000","last_clean":"2026-02-22T22:38:17.576464+0000","last_became_active":"2026-02-22T22:38:15.574673+0000","last_became_peered":"2026-02-22T22:38:15.574673+0000","last_unstale":"2026-02-22T22:38:17.576464+0000","last_undegraded":"2026-02-22T22:38:17.576464+0000","last_fullsized":"2026-02-22T22:38:17.576464+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:46.550560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031834000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.576467+0000","last_change":"2026-02-22T22:38:17.576467+0000","last_active":"2026-02-22T22:38:17.576467+0000","last_peered":"2026-02-22T22:38:17.576467+0000","last_clean":"2026-02-22T22:38:17.576467+0000","last_became_active":"2026-02-22T22:38:15.574763+0000","last_became_peered":"2026-02-22T22:38:15.574763+0000","last_unstale":"2026-02-22T22:38:17.576467+0000","last_undegraded":"2026-02-22T22:38:17.576467+0000","last_fullsized":"2026-02-22T22:38:17.576467+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:25.430048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031511400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"15'2","reported_seq":21,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.575570+0000","last_change":"2026-02-22T22:38:17.575058+0000","last_active":"2026-02-22T22:38:17.575570+0000","last_peered":"2026-02-22T22:38:17.575570+0000","last_clean":"2026-02-22T22:38:17.575570+0000","last_became_active":"2026-02-22T22:38:15.574409+0000","last_became_peered":"2026-02-22T22:38:15.574409+0000","last_unstale":"2026-02-22T22:38:17.575570+0000","last_undegraded":"2026-02-22T22:38:17.575570+0000","last_fullsized":"2026-02-22T22:38:17.575570+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:32:34.479960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053028299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.677203+0000","last_change":"2026-02-22T22:38:17.677203+0000","last_active":"2026-02-22T22:38:17.677203+0000","last_peered":"2026-02-22T22:38:17.677203+0000","last_clean":"2026-02-22T22:38:17.677203+0000","last_became_active":"2026-02-22T22:38:15.574389+0000","last_became_peered":"2026-02-22T22:38:15.574389+0000","last_unstale":"2026-02-22T22:38:17.677203+0000","last_undegraded":"2026-02-22T22:38:17.677203+0000","last_fullsized":"2026-02-22T22:38:17.677203+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:09:24.494810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021430700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.677186+0000","last_change":"2026-02-22T22:38:17.677186+0000","last_active":"2026-02-22T22:38:17.677186+0000","last_peered":"2026-02-22T22:38:17.677186+0000","last_clean":"2026-02-22T22:38:17.677186+0000","last_became_active":"2026-02-22T22:38:15.574402+0000","last_became_peered":"2026-02-22T22:38:15.574402+0000","last_unstale":"2026-02-22T22:38:17.677186+0000","last_undegraded":"2026-02-22T22:38:17.677186+0000","last_fullsized":"2026-02-22T22:38:17.677186+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:19.472593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019295799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"13'1","reported_seq":20,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.576831+0000","last_change":"2026-02-22T22:38:17.576831+0000","last_active":"2026-02-22T22:38:17.576831+0000","last_peered":"2026-02-22T22:38:17.576831+0000","last_clean":"2026-02-22T22:38:17.576831+0000","last_became_active":"2026-02-22T22:38:15.574263+0000","last_became_peered":"2026-02-22T22:38:15.574263+0000","last_unstale":"2026-02-22T22:38:17.576831+0000","last_undegraded":"2026-02-22T22:38:17.576831+0000","last_fullsized":"2026-02-22T22:38:17.576831+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:41.940013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068121299999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":15,"state":"active+clean","last_fresh":"2026-02-22T22:38:17.575933+0000","last_change":"2026-02-22T22:38:11.502395+0000","last_active":"2026-02-22T22:38:17.575933+0000","last_peered":"2026-02-22T22:38:17.575933+0000","last_clean":"2026-02-22T22:38:17.575933+0000","last_became_active":"2026-02-22T22:38:11.501894+0000","last_became_peered":"2026-02-22T22:38:11.501894+0000","last_unstale":"2026-02-22T22:38:17.575933+0000","last_undegraded":"2026-02-22T22:38:17.575933+0000","last_fullsized":"2026-02-22T22:38:17.575933+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_clean_scrub_stamp":"2026-02-22T22:38:09.558114+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:19:00.206795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705668,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27080,"kb_used_data":112,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344760,"statfs":{"total":96636764160,"available":96609034240,"internally_reserved":0,"allocated":114688,"data_stored":31634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":720896,"data_stored":625486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27672,"kb_used_data":704,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344168,"statfs":{"total":96636764160,"available":96608428032,"internally_reserved":0,"allocated":720896,"data_stored":625486,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:38:21.078 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:38:21.078 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-22T22:38:21.078 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-22T22:38:21.078 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-22T22:38:21.258 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:38:21.258 INFO:teuthology.orchestra.run.trial178.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T22:38:21.267 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-22T22:38:21.267 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T22:38:21.285 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T22:38:21.285 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:38:21.285 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:21.481 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.4] 2026-02-22T22:38:21.481 INFO:teuthology.orchestra.run.trial178.stdout: key = AQANhZtpXD04LhAAnGaH2U5HZe/z7Q4XZhK4Hw== 2026-02-22T22:38:21.481 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:21.481 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:21.482 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.4 2026-02-22T22:38:21.490 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:21.685 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.5] 2026-02-22T22:38:21.685 INFO:teuthology.orchestra.run.trial178.stdout: key = AQANhZtpgpdeMxAAebAPohhR7T8wK2Jd9w1ETw== 2026-02-22T22:38:21.685 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:21.685 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:21.685 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.5 2026-02-22T22:38:21.694 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:21.891 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.6] 2026-02-22T22:38:21.891 INFO:teuthology.orchestra.run.trial178.stdout: key = AQANhZtpGalaOBAAaiN4kCj4TU4V681+21JqPg== 2026-02-22T22:38:21.891 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:21.892 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:21.892 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.6 2026-02-22T22:38:21.901 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:22.098 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.4] 2026-02-22T22:38:22.098 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpRHZHKBAArGUqNt76mmUtH6pyUVXYXA== 2026-02-22T22:38:22.098 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:22.098 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:22.098 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.4 2026-02-22T22:38:22.108 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:22.302 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.5] 2026-02-22T22:38:22.302 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpQyxGLRAARJYkVzbX+KZeONXSkgXMvQ== 2026-02-22T22:38:22.302 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:22.303 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:22.303 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.5 2026-02-22T22:38:22.312 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:22.508 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.6] 2026-02-22T22:38:22.509 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpGmlMMhAAr10b8YosCw0J9S+hc08ZUQ== 2026-02-22T22:38:22.509 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:22.509 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:22.509 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.6 2026-02-22T22:38:22.518 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T22:38:22.530 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T22:38:22.530 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:38:22.531 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:22.729 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror] 2026-02-22T22:38:22.729 INFO:teuthology.orchestra.run.trial178.stdout: key = AQANhZtpRR2uFBAALH/abVpa4XAbhBMSSfYGDg== 2026-02-22T22:38:22.729 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T22:38:22.729 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:22.729 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror 2026-02-22T22:38:22.738 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:22.936 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.0] 2026-02-22T22:38:22.936 INFO:teuthology.orchestra.run.trial178.stdout: key = AQANhZtpbtvDGRAAMoT1/e9FgKJZ6s0QmmitVg== 2026-02-22T22:38:22.936 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:22.936 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:22.936 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.0 2026-02-22T22:38:22.945 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:23.141 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.1] 2026-02-22T22:38:23.142 INFO:teuthology.orchestra.run.trial178.stdout: key = AQANhZtp/b/6HhAAHxFTXhm3wQEGf7H6QOTk+A== 2026-02-22T22:38:23.142 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:23.142 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:23.142 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.1 2026-02-22T22:38:23.151 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:23.348 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.2] 2026-02-22T22:38:23.348 INFO:teuthology.orchestra.run.trial178.stdout: key = AQANhZtp/bYqJBAARWCLeV/3o4JxeZusEyGjkw== 2026-02-22T22:38:23.348 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:23.348 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:23.349 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.2 2026-02-22T22:38:23.357 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:23.554 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.3] 2026-02-22T22:38:23.554 INFO:teuthology.orchestra.run.trial178.stdout: key = AQANhZtprJotKRAAxxhsJDyFX3DNyj+CWoS2Rw== 2026-02-22T22:38:23.554 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:23.554 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:23.555 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.3 2026-02-22T22:38:23.564 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:23.757 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror] 2026-02-22T22:38:23.758 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtppl19DhAAeg0XJcUi0i2h/XEsl7RzEw== 2026-02-22T22:38:23.758 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T22:38:23.758 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:23.758 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror 2026-02-22T22:38:23.768 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:23.963 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.0] 2026-02-22T22:38:23.963 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpWb8YFBAAdBXYRn6iqO6TICYIN4wBKw== 2026-02-22T22:38:23.964 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:23.964 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:23.964 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.0 2026-02-22T22:38:23.973 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:24.170 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.1] 2026-02-22T22:38:24.170 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpg4UaGRAAvSm1zYDrzFMEkJQKeCcQ+g== 2026-02-22T22:38:24.170 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:24.170 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:24.171 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.1 2026-02-22T22:38:24.180 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:24.374 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.2] 2026-02-22T22:38:24.374 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtpz2kuHhAAYMwTRzJb5CeUCQLpq2R3iQ== 2026-02-22T22:38:24.375 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:24.375 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:24.375 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.2 2026-02-22T22:38:24.384 DEBUG:teuthology.orchestra.run.trial178:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:38:24.583 INFO:teuthology.orchestra.run.trial178.stdout:[client.mirror.3] 2026-02-22T22:38:24.583 INFO:teuthology.orchestra.run.trial178.stdout: key = AQA2hZtplBU0IxAAyT4ImepXFH6xsnLghjdKUw== 2026-02-22T22:38:24.583 INFO:teuthology.orchestra.run.trial178.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:38:24.583 INFO:teuthology.orchestra.run.trial178.stdout: caps osd = "profile rbd" 2026-02-22T22:38:24.584 INFO:teuthology.orchestra.run.trial178.stderr:updated caps for client.mirror.3 2026-02-22T22:38:24.594 INFO:teuthology.run_tasks:Running task workunit... 2026-02-22T22:38:24.621 INFO:tasks.workunit:Pulling workunits from ref f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:38:24.622 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-22T22:38:24.622 DEBUG:teuthology.orchestra.run.trial178:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:38:24.639 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:38:24.639 INFO:teuthology.orchestra.run.trial178.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-22T22:38:24.639 DEBUG:teuthology.orchestra.run.trial178:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:38:24.695 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:38:24.696 DEBUG:teuthology.orchestra.run.trial178:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-22T22:38:24.751 INFO:tasks.workunit:timeout=3h 2026-02-22T22:38:24.752 INFO:tasks.workunit:cleanup=True 2026-02-22T22:38:24.752 DEBUG:teuthology.orchestra.run.trial178:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:38:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-22T22:38:45.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:Note: switching to 'f90f4eaebb28b619cf76e80cbbe997656da69579'. 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:state without impacting any branches by switching back to a branch. 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: git switch -c 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: 2026-02-22T22:38:45.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:Or undo this operation with: 2026-02-22T22:38:45.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: 2026-02-22T22:38:45.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: git switch - 2026-02-22T22:38:45.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: 2026-02-22T22:38:45.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-22T22:38:45.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: 2026-02-22T22:38:45.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:HEAD is now at f90f4eaebb2 Merge branch 'wip-74712-squid' of https://github.com/idryomov/ceph into wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:38:45.732 DEBUG:teuthology.orchestra.run.trial178:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-22T22:38:45.790 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-22T22:38:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T22:38:45.792 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-22T22:38:45.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-22T22:38:45.919 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-22T22:38:45.945 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T22:38:45.947 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T22:38:45.947 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-22T22:38:45.970 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T22:38:45.974 DEBUG:teuthology.orchestra.run.trial178:> set -ex 2026-02-22T22:38:45.974 DEBUG:teuthology.orchestra.run.trial178:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-22T22:38:46.032 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_bootstrap.sh on cluster1.client.mirror... 2026-02-22T22:38:46.035 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_bootstrap.sh... 2026-02-22T22:38:46.035 DEBUG:teuthology.orchestra.run.trial178:workunit test rbd/rbd_mirror_bootstrap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f90f4eaebb28b619cf76e80cbbe997656da69579 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=image RBD_MIRROR_INSTANCES=1 RBD_MIRROR_MODE=snapshot RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_bootstrap.sh 2026-02-22T22:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ RBD_MIRROR_MANUAL_PEERS=1 2026-02-22T22:38:46.092 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ RBD_MIRROR_INSTANCES=1 2026-02-22T22:38:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_bootstrap.sh 2026-02-22T22:38:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-22T22:38:46.094 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ type xmlstarlet 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ XMLSTARLET=xmlstarlet 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ RBD_MIRROR_INSTANCES=1 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CLUSTER1=cluster1 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CLUSTER2=cluster2 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ POOL=mirror 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ PARENT_POOL=mirror_parent 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ NS1=ns1 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ NS2=ns2 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ TEMPDIR= 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ID=mirror 2026-02-22T22:38:46.095 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ RBD_MIRROR_MODE=snapshot 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ MIRROR_POOL_MODE=image 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' snapshot = snapshot ']' 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ MIRROR_POOL_MODE=image 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS='--id mirror' 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ LAST_MIRROR_INSTANCE=0 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_bootstrap.sh 2026-02-22T22:38:46.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-22T22:38:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-22T22:38:46.098 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_BIN=. 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_SRC=. 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 -gt 0 ']' 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ setup 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local c 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ setup_tempdir 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mktemp -d 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ TEMPDIR=/tmp/tmp.AySj3raLFU 2026-02-22T22:38:46.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -z 1 ']' 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ setup_pools cluster1 cluster2 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local remote_cluster=cluster2 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mon_map_file 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mon_addr 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local admin_key_file 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local uuid 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:46.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-22T22:38:46.735 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:pool 'mirror' already exists 2026-02-22T22:38:46.741 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:46.741 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-22T22:38:47.737 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:pool 'mirror_parent' already exists 2026-02-22T22:38:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:47.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-22T22:38:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:50.705 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-22T22:38:53.710 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:38:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:53.711 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror image 2026-02-22T22:38:53.738 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-22T22:38:53.764 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-22T22:38:53.790 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-22T22:38:53.813 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-22T22:38:53.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 image 2026-02-22T22:38:53.863 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-22T22:38:53.890 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 image 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -z 1 ']' 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ setup_pools cluster2 cluster1 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local remote_cluster=cluster1 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mon_map_file 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mon_addr 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local admin_key_file 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local uuid 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:53.915 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-22T22:38:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:pool 'mirror' already exists 2026-02-22T22:38:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:54.556 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-22T22:38:55.552 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:pool 'mirror_parent' already exists 2026-02-22T22:38:55.561 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:55.561 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-22T22:38:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:38:58.514 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-22T22:39:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:39:01.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror image 2026-02-22T22:39:01.546 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-22T22:39:01.570 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-22T22:39:01.592 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-22T22:39:01.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-22T22:39:01.636 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 image 2026-02-22T22:39:01.661 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-22T22:39:01.684 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 image 2026-02-22T22:39:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -z 1 ']' 2026-02-22T22:39:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ testlog 'TEST: bootstrap cluster2 from cluster1' 2026-02-22T22:39:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ tee -a /tmp/tmp.AySj3raLFU/rbd-mirror.test.log 2026-02-22T22:39:01.709 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ date '+%F %T' 2026-02-22T22:39:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo 2026-02-22 22:39:01 TEST: bootstrap cluster2 from cluster1 2026-02-22T22:39:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:2026-02-22 22:39:01 TEST: bootstrap cluster2 from cluster1 2026-02-22T22:39:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ TOKEN=/tmp/tmp.AySj3raLFU/peer-token 2026-02-22T22:39:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ TOKEN_2=/tmp/tmp.AySj3raLFU/peer-token-2 2026-02-22T22:39:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:39:01.710 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror pool peer bootstrap create mirror 2026-02-22T22:39:01.735 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:39:01.735 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror pool peer bootstrap create mirror_parent 2026-02-22T22:39:01.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ cmp /tmp/tmp.AySj3raLFU/peer-token /tmp/tmp.AySj3raLFU/peer-token-2 2026-02-22T22:39:01.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:39:01.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 --pool mirror mirror pool peer bootstrap import /tmp/tmp.AySj3raLFU/peer-token --direction rx-only 2026-02-22T22:39:01.995 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:39:01.996 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 --pool mirror_parent mirror pool peer bootstrap import /tmp/tmp.AySj3raLFU/peer-token --direction rx-tx 2026-02-22T22:39:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ start_mirrors cluster1 2026-02-22T22:39:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:02.033 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ seq 0 0 2026-02-22T22:39:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ start_mirror cluster1:0 2026-02-22T22:39:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1:0 2026-02-22T22:39:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local instance 2026-02-22T22:39:02.035 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local val=cluster1:0 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster_var_name=cluster 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local instance_var_name=instance 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ cluster=cluster1 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ instance=0 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ eval cluster=cluster1 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster1 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ eval instance=0 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:39:02.036 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:39:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ start_mirrors cluster2 2026-02-22T22:39:02.053 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:02.054 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ seq 0 0 2026-02-22T22:39:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ start_mirror cluster2:0 2026-02-22T22:39:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2:0 2026-02-22T22:39:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local instance 2026-02-22T22:39:02.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local val=cluster2:0 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster_var_name=cluster 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local instance_var_name=instance 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ cluster=cluster2 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ instance=0 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ eval cluster=cluster2 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ eval instance=0 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:39:02.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:39:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ testlog 'TEST: verify rx-only direction' 2026-02-22T22:39:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ tee -a /tmp/tmp.AySj3raLFU/rbd-mirror.test.log 2026-02-22T22:39:02.074 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ date '+%F %T' 2026-02-22T22:39:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo 2026-02-22 22:39:02 TEST: verify rx-only direction 2026-02-22T22:39:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:2026-02-22 22:39:02 TEST: verify rx-only direction 2026-02-22T22:39:02.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 --pool mirror mirror pool info --format json 2026-02-22T22:39:02.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.peers[0].direction == "rx-only"' 2026-02-22T22:39:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:02.099 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 --pool mirror mirror pool info --format json 2026-02-22T22:39:02.100 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.peers | length > 0' 2026-02-22T22:39:02.126 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 --pool mirror mirror pool info --format json 2026-02-22T22:39:02.127 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.peers[0].direction == "tx-only"' 2026-02-22T22:39:02.153 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image_and_enable_mirror cluster1 mirror image1 2026-02-22T22:39:02.154 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image cluster1 mirror image1 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:02.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size=128 2026-02-22T22:39:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:02.156 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/image1 2026-02-22T22:39:02.186 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:39:02.186 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:02.186 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' image = image ']' 2026-02-22T22:39:02.186 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:02.186 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ enable_mirror cluster1 mirror image1 snapshot 2026-02-22T22:39:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:02.187 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/image1 snapshot 2026-02-22T22:39:03.042 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Mirroring enabled 2026-02-22T22:39:03.047 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster1 info mirror/image1 2026-02-22T22:39:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:rbd image 'image1': 2026-02-22T22:39:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: size 128 MiB in 32 objects 2026-02-22T22:39:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: order 22 (4 MiB objects) 2026-02-22T22:39:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: snapshot_count: 1 2026-02-22T22:39:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: id: 10bb64ece3b4 2026-02-22T22:39:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: block_name_prefix: rbd_data.10bb64ece3b4 2026-02-22T22:39:03.075 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: format: 2 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: op_features: 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: flags: 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: create_timestamp: Sun Feb 22 22:39:02 2026 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: access_timestamp: Sun Feb 22 22:39:02 2026 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: modify_timestamp: Sun Feb 22 22:39:02 2026 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: journal: 10bb64ece3b4 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring state: enabled 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring mode: snapshot 2026-02-22T22:39:03.076 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring global id: 797f5a58-34c9-4532-af30-cce903e2f787 2026-02-22T22:39:03.077 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring primary: true 2026-02-22T22:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image_and_enable_mirror cluster1 mirror/ns1 image1 2026-02-22T22:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:03.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image cluster1 mirror/ns1 image1 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size=128 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:03.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/ns1/image1 2026-02-22T22:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-22T22:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' image = image ']' 2026-02-22T22:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ enable_mirror cluster1 mirror/ns1 image1 snapshot 2026-02-22T22:39:03.111 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:03.112 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster1 mirror image enable mirror/ns1/image1 snapshot 2026-02-22T22:39:03.684 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Mirroring enabled 2026-02-22T22:39:03.688 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster1 info mirror/ns1/image1 2026-02-22T22:39:03.715 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:rbd image 'image1': 2026-02-22T22:39:03.715 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: size 128 MiB in 32 objects 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: order 22 (4 MiB objects) 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: snapshot_count: 1 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: id: 10c4e203018 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: block_name_prefix: rbd_data.10c4e203018 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: format: 2 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: op_features: 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: flags: 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: create_timestamp: Sun Feb 22 22:39:03 2026 2026-02-22T22:39:03.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: access_timestamp: Sun Feb 22 22:39:03 2026 2026-02-22T22:39:03.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: modify_timestamp: Sun Feb 22 22:39:03 2026 2026-02-22T22:39:03.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: journal: 10c4e203018 2026-02-22T22:39:03.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring state: enabled 2026-02-22T22:39:03.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring mode: snapshot 2026-02-22T22:39:03.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring global id: cdda9386-acd6-4063-b296-244fb50737e3 2026-02-22T22:39:03.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring primary: true 2026-02-22T22:39:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_started cluster2 mirror image1 2026-02-22T22:39:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:03.718 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_state cluster2 mirror image1 started 2026-02-22T22:39:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=started 2026-02-22T22:39:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local s 2026-02-22T22:39:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:03.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:04.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_image_replay_state cluster2 mirror image1 started 2026-02-22T22:39:04.719 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local test_state=started 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_result 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local current_state=stopped 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemons cluster2 rbd mirror status mirror/image1 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_instance=cluster2 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance=cluster2 2026-02-22T22:39:04.720 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local loop_instance 2026-02-22T22:39:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ grep -i state 2026-02-22T22:39:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:04.721 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sleep 0 2026-02-22T22:39:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:39:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ seq 0 0 2026-02-22T22:39:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/image1 2026-02-22T22:39:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2:0 2026-02-22T22:39:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:39:04.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster2:0 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster2 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:39:04.725 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster2 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster2:0 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ local_cluster=cluster2 2026-02-22T22:39:04.726 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:39:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:39:04.727 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:39:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:04.735 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:04.736 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/image1 2026-02-22T22:39:04.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:Invalid command: unused arguments: ['mirror/image1'] 2026-02-22T22:39:04.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:39:04.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:admin_socket: invalid command 2026-02-22T22:39:04.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:04.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sleep 1 2026-02-22T22:39:05.779 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:39:05.780 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ seq 0 0 2026-02-22T22:39:05.782 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/image1 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2:0 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster2:0 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:05.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster2 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster2 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster2:0 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:39:05.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:39:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ local_cluster=cluster2 2026-02-22T22:39:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:39:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:39:05.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:39:05.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:05.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:05.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:05.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/image1 2026-02-22T22:39:05.842 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ return 0 2026-02-22T22:39:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:39:05.843 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:39:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep -i Replaying 2026-02-22T22:39:05.844 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: "state": "Replaying" 2026-02-22T22:39:05.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ current_state=started 2026-02-22T22:39:05.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test started = started 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ write_image cluster1 mirror image1 100 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local count=100 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size= 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ size=4096 2026-02-22T22:39:05.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 bench mirror/image1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:39:05.881 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:39:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:39:05.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:elapsed: 0 ops: 100 ops/sec: 7692.59 bytes/sec: 30 MiB/s 2026-02-22T22:39:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror image1 2026-02-22T22:39:05.899 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster2 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ mirror_image_snapshot cluster1 mirror image1 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:05.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/image1 2026-02-22T22:39:06.042 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Snapshot ID: 5 2026-02-22T22:39:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror image1 2026-02-22T22:39:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster2 2026-02-22T22:39:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:06.048 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster1-mirror-image1.status 2026-02-22T22:39:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster1-mirror-image1.status 2026-02-22T22:39:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:06.049 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_log=/tmp/tmp.AySj3raLFU/cluster1-mirror-image1.status 2026-02-22T22:39:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster2-mirror-image1.status 2026-02-22T22:39:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster2-mirror-image1.status 2026-02-22T22:39:06.050 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_status_log=/tmp/tmp.AySj3raLFU/cluster2-mirror-image1.status 2026-02-22T22:39:06.051 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster1 mirror image1 /tmp/tmp.AySj3raLFU/cluster1-mirror-image1.status 2026-02-22T22:39:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster1-mirror-image1.status 2026-02-22T22:39:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 snap list --all mirror/image1 --format xml 2026-02-22T22:39:06.052 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:06.081 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:39:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local snapshot_id=5 2026-02-22T22:39:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ true 2026-02-22T22:39:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:06.083 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.2 2026-02-22T22:39:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror image1 /tmp/tmp.AySj3raLFU/cluster2-mirror-image1.status 2026-02-22T22:39:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:06.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror-image1.status 2026-02-22T22:39:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror/image1 --format xml 2026-02-22T22:39:06.284 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:06.312 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=3 2026-02-22T22:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 5 = 3 2026-02-22T22:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:06.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.4 2026-02-22T22:39:06.715 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror image1 /tmp/tmp.AySj3raLFU/cluster2-mirror-image1.status 2026-02-22T22:39:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror-image1.status 2026-02-22T22:39:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror/image1 --format xml 2026-02-22T22:39:06.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:06.745 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:06.748 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=5 2026-02-22T22:39:06.748 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 5 = 5 2026-02-22T22:39:06.748 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:06.748 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror image1 2026-02-22T22:39:06.748 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:06.748 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image1 up+replaying 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:06.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_status_in_pool_dir cluster2 mirror image1 up+replaying '' '' 2026-02-22T22:39:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:07.749 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:07.750 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:07.750 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:07.750 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror image status mirror/image1 2026-02-22T22:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='image1: 2026-02-22T22:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: global_id: 797f5a58-34c9-4532-af30-cce903e2f787 2026-02-22T22:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+replaying 2026-02-22T22:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799943,"replay_state":"idle"} 2026-02-22T22:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: service: mirror.0 on trial178 2026-02-22T22:39:07.780 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:05' 2026-02-22T22:39:07.781 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:39:07.781 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: state: up+replaying 2026-02-22T22:39:07.781 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ description: .*' 2026-02-22T22:39:07.782 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799943,"replay_state":"idle"} 2026-02-22T22:39:07.782 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:07.782 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:07.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^up+' 2026-02-22T22:39:07.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:07.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:39:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: service: mirror.0 on trial178 2026-02-22T22:39:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update 2026-02-22T22:39:07.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:39:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:05' 2026-02-22T22:39:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image1 up+replaying '2026-02-22 22:39:05' 2026-02-22T22:39:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror 2026-02-22T22:39:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:07.786 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local 'prev_last_update=2026-02-22 22:39:05' 2026-02-22T22:39:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:07.787 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:39:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='OKOKOK142654270mirror.0trial17819.2.3-689-gf90f4eaetrueOKimage1797f5a58-34c9-4532-af30-cce903e2f787up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799943,"replay_state":"idle"}42654270mirror.0trial1782026-02-22 22:39:05' 2026-02-22T22:39:07.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update state 2026-02-22T22:39:07.816 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/last_update' 2026-02-22T22:39:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:05' 2026-02-22T22:39:07.817 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/state' 2026-02-22T22:39:07.818 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ state=up+replaying 2026-02-22T22:39:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:07.819 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep up+replaying 2026-02-22T22:39:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:07.820 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster1 mirror 2026-02-22T22:39:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror 2026-02-22T22:39:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:07.821 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format json 2026-02-22T22:39:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"WARNING","daemon_health":"WARNING","image_health":"OK","states":{"replaying":1}},"daemons":[{"service_id":"4268","instance_id":"","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"OK"}],"images":[{"name":"image1","global_id":"797f5a58-34c9-4532-af30-cce903e2f787"}]}' 2026-02-22T22:39:07.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.states == {"replaying": 1}' 2026-02-22T22:39:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror 2026-02-22T22:39:07.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror 2026-02-22T22:39:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:07.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format json 2026-02-22T22:39:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":1}},"daemons":[{"service_id":"4265","instance_id":"4270","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"797f5a58-34c9-4532-af30-cce903e2f787","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799943,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:05"}]}' 2026-02-22T22:39:07.884 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.states == {"replaying": 1}' 2026-02-22T22:39:07.885 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:07.886 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_started cluster2 mirror/ns1 image1 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_state cluster2 mirror/ns1 image1 started 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=started 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local s 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:07.887 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_image_replay_state cluster2 mirror/ns1 image1 started 2026-02-22T22:39:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local test_state=started 2026-02-22T22:39:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_result 2026-02-22T22:39:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local current_state=stopped 2026-02-22T22:39:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemons cluster2 rbd mirror status mirror/ns1/image1 2026-02-22T22:39:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ grep -i state 2026-02-22T22:39:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_instance=cluster2 2026-02-22T22:39:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance=cluster2 2026-02-22T22:39:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local loop_instance 2026-02-22T22:39:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sleep 0 2026-02-22T22:39:08.892 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ seq 0 0 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/ns1/image1 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2:0 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster2:0 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:39:08.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster2 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster2 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:39:08.894 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster2:0 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ local_cluster=cluster2 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:39:08.895 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:39:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:08.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/ns1/image1 2026-02-22T22:39:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ return 0 2026-02-22T22:39:08.950 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:39:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:39:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep -i Replaying 2026-02-22T22:39:08.951 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: "state": "Replaying" 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ current_state=started 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test started = started 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ write_image cluster1 mirror/ns1 image1 100 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local count=100 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size= 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:39:08.953 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ size=4096 2026-02-22T22:39:08.954 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 bench mirror/ns1/image1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:39:08.989 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:39:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:39:09.000 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:elapsed: 0 ops: 100 ops/sec: 8333.64 bytes/sec: 33 MiB/s 2026-02-22T22:39:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror/ns1 image1 2026-02-22T22:39:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster2 2026-02-22T22:39:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:09.007 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:39:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ mirror_image_snapshot cluster1 mirror/ns1 image1 2026-02-22T22:39:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:09.008 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror/ns1/image1 2026-02-22T22:39:10.049 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Snapshot ID: 6 2026-02-22T22:39:10.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror/ns1 image1 2026-02-22T22:39:10.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster2 2026-02-22T22:39:10.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster1-mirror/ns1-image1.status 2026-02-22T22:39:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster1-mirror/ns1-image1.status 2026-02-22T22:39:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:10.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_log=/tmp/tmp.AySj3raLFU/cluster1-mirror_ns1-image1.status 2026-02-22T22:39:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster2-mirror/ns1-image1.status 2026-02-22T22:39:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster2-mirror/ns1-image1.status 2026-02-22T22:39:10.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_status_log=/tmp/tmp.AySj3raLFU/cluster2-mirror_ns1-image1.status 2026-02-22T22:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster1 mirror/ns1 image1 /tmp/tmp.AySj3raLFU/cluster1-mirror_ns1-image1.status 2026-02-22T22:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster1-mirror_ns1-image1.status 2026-02-22T22:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 snap list --all mirror/ns1/image1 --format xml 2026-02-22T22:39:10.059 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:10.089 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:39:10.090 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local snapshot_id=6 2026-02-22T22:39:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ true 2026-02-22T22:39:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:10.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.2 2026-02-22T22:39:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 image1 /tmp/tmp.AySj3raLFU/cluster2-mirror_ns1-image1.status 2026-02-22T22:39:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror_ns1-image1.status 2026-02-22T22:39:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/image1 --format xml 2026-02-22T22:39:10.292 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:10.320 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=4 2026-02-22T22:39:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 6 = 4 2026-02-22T22:39:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.4 2026-02-22T22:39:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror/ns1 image1 /tmp/tmp.AySj3raLFU/cluster2-mirror_ns1-image1.status 2026-02-22T22:39:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror_ns1-image1.status 2026-02-22T22:39:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror/ns1/image1 --format xml 2026-02-22T22:39:10.724 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:10.753 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=6 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 6 = 6 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror/ns1 image1 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_status_in_pool_dir cluster2 mirror/ns1 image1 up+replaying 2026-02-22T22:39:10.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:10.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_status_in_pool_dir cluster2 mirror/ns1 image1 up+replaying '' '' 2026-02-22T22:39:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:11.757 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:11.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror image status mirror/ns1/image1 2026-02-22T22:39:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='image1: 2026-02-22T22:39:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: global_id: cdda9386-acd6-4063-b296-244fb50737e3 2026-02-22T22:39:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+replaying 2026-02-22T22:39:11.787 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799943,"replay_state":"idle"} 2026-02-22T22:39:11.788 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: service: mirror.0 on trial178 2026-02-22T22:39:11.788 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:05' 2026-02-22T22:39:11.788 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:39:11.788 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: state: up+replaying 2026-02-22T22:39:11.789 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ description: .*' 2026-02-22T22:39:11.789 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799943,"replay_state":"idle"} 2026-02-22T22:39:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^up+' 2026-02-22T22:39:11.790 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:39:11.791 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: service: mirror.0 on trial178 2026-02-22T22:39:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update 2026-02-22T22:39:11.792 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:39:11.793 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:05' 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_mirror_pool_status_verbose cluster2 mirror/ns1 image1 up+replaying '2026-02-22 22:39:05' 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror/ns1 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local 'prev_last_update=2026-02-22 22:39:05' 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:11.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror/ns1 --verbose --format xml 2026-02-22T22:39:11.822 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='OKOKOK142654270mirror.0trial17819.2.3-689-gf90f4eaetrueOKimage1cdda9386-acd6-4063-b296-244fb50737e3up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799943,"replay_state":"idle"}42654270mirror.0trial1782026-02-22 22:39:05' 2026-02-22T22:39:11.822 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update state 2026-02-22T22:39:11.822 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/last_update' 2026-02-22T22:39:11.823 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:05' 2026-02-22T22:39:11.824 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/state' 2026-02-22T22:39:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ state=up+replaying 2026-02-22T22:39:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:11.826 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep up+replaying 2026-02-22T22:39:11.827 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster1 mirror/ns1 2026-02-22T22:39:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror/ns1 2026-02-22T22:39:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:11.828 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format json 2026-02-22T22:39:11.858 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"WARNING","daemon_health":"WARNING","image_health":"OK","states":{"replaying":1}},"daemons":[{"service_id":"4268","instance_id":"","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"OK"}],"images":[{"name":"image1","global_id":"cdda9386-acd6-4063-b296-244fb50737e3"}]}' 2026-02-22T22:39:11.858 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.states == {"replaying": 1}' 2026-02-22T22:39:11.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror/ns1 2026-02-22T22:39:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror/ns1 2026-02-22T22:39:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:11.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror/ns1 --verbose --format json 2026-02-22T22:39:11.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":1}},"daemons":[{"service_id":"4265","instance_id":"4270","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"cdda9386-acd6-4063-b296-244fb50737e3","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799943,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:05"}]}' 2026-02-22T22:39:11.889 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.states == {"replaying": 1}' 2026-02-22T22:39:11.890 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ testlog 'TEST: verify rx-tx direction' 2026-02-22T22:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ tee -a /tmp/tmp.AySj3raLFU/rbd-mirror.test.log 2026-02-22T22:39:11.891 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ date '+%F %T' 2026-02-22T22:39:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo 2026-02-22 22:39:11 TEST: verify rx-tx direction 2026-02-22T22:39:11.892 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:2026-02-22 22:39:11 TEST: verify rx-tx direction 2026-02-22T22:39:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 --pool mirror_parent mirror pool info --format json 2026-02-22T22:39:11.893 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.peers[0].direction == "rx-tx"' 2026-02-22T22:39:11.917 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:11.918 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 --pool mirror_parent mirror pool info --format json 2026-02-22T22:39:11.918 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.peers[0].direction == "rx-tx"' 2026-02-22T22:39:11.940 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image_and_enable_mirror cluster1 mirror_parent image1 2026-02-22T22:39:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:11.942 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image cluster1 mirror_parent image1 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:11.943 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size=128 2026-02-22T22:39:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:11.944 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/image1 2026-02-22T22:39:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-02-22T22:39:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:11.973 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' image = image ']' 2026-02-22T22:39:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ enable_mirror cluster1 mirror_parent image1 snapshot 2026-02-22T22:39:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:11.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster1 mirror image enable mirror_parent/image1 snapshot 2026-02-22T22:39:12.052 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Mirroring enabled 2026-02-22T22:39:12.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster1 info mirror_parent/image1 2026-02-22T22:39:12.083 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:rbd image 'image1': 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: size 128 MiB in 32 objects 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: order 22 (4 MiB objects) 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: snapshot_count: 1 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: id: 10e6a4359028 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: block_name_prefix: rbd_data.10e6a4359028 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: format: 2 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: op_features: 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: flags: 2026-02-22T22:39:12.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: create_timestamp: Sun Feb 22 22:39:11 2026 2026-02-22T22:39:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: access_timestamp: Sun Feb 22 22:39:11 2026 2026-02-22T22:39:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: modify_timestamp: Sun Feb 22 22:39:11 2026 2026-02-22T22:39:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: journal: 10e6a4359028 2026-02-22T22:39:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring state: enabled 2026-02-22T22:39:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring mode: snapshot 2026-02-22T22:39:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring global id: b62f7034-d36e-473d-8201-0c6d09c052f8 2026-02-22T22:39:12.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring primary: true 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent image2 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:12.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image cluster2 mirror_parent image2 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size=128 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:12.088 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/image2 2026-02-22T22:39:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-02-22T22:39:12.116 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' image = image ']' 2026-02-22T22:39:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ enable_mirror cluster2 mirror_parent image2 snapshot 2026-02-22T22:39:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:12.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/image2 snapshot 2026-02-22T22:39:12.584 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Mirroring enabled 2026-02-22T22:39:12.589 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster2 info mirror_parent/image2 2026-02-22T22:39:12.613 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:rbd image 'image2': 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: size 128 MiB in 32 objects 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: order 22 (4 MiB objects) 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: snapshot_count: 1 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: id: 10d8969c8e8a 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: block_name_prefix: rbd_data.10d8969c8e8a 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: format: 2 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: op_features: 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: flags: 2026-02-22T22:39:12.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: create_timestamp: Sun Feb 22 22:39:12 2026 2026-02-22T22:39:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: access_timestamp: Sun Feb 22 22:39:12 2026 2026-02-22T22:39:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: modify_timestamp: Sun Feb 22 22:39:12 2026 2026-02-22T22:39:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: journal: 10d8969c8e8a 2026-02-22T22:39:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring state: enabled 2026-02-22T22:39:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring mode: snapshot 2026-02-22T22:39:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring global id: cd82a87a-1ebd-4f76-98ef-2422e809f845 2026-02-22T22:39:12.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring primary: true 2026-02-22T22:39:12.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image_and_enable_mirror cluster1 mirror_parent/ns1 image1 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image cluster1 mirror_parent/ns1 image1 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:12.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:12.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:12.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:12.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size=128 2026-02-22T22:39:12.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:12.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/ns1/image1 2026-02-22T22:39:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ is_pool_mirror_mode_image mirror_parent/ns1 2026-02-22T22:39:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:12.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' image = image ']' 2026-02-22T22:39:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ enable_mirror cluster1 mirror_parent/ns1 image1 snapshot 2026-02-22T22:39:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:12.650 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster1 mirror image enable mirror_parent/ns1/image1 snapshot 2026-02-22T22:39:13.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Mirroring enabled 2026-02-22T22:39:13.060 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster1 info mirror_parent/ns1/image1 2026-02-22T22:39:13.084 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:rbd image 'image1': 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: size 128 MiB in 32 objects 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: order 22 (4 MiB objects) 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: snapshot_count: 1 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: id: 10ed5f6cd2c2 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: block_name_prefix: rbd_data.10ed5f6cd2c2 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: format: 2 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: op_features: 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: flags: 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: create_timestamp: Sun Feb 22 22:39:12 2026 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: access_timestamp: Sun Feb 22 22:39:12 2026 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: modify_timestamp: Sun Feb 22 22:39:12 2026 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: journal: 10ed5f6cd2c2 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring state: enabled 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring mode: snapshot 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring global id: c1154afb-c3df-43a9-8813-a4d98bf6cbfc 2026-02-22T22:39:13.085 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring primary: true 2026-02-22T22:39:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent/ns1 image2 2026-02-22T22:39:13.086 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ create_image cluster2 mirror_parent/ns1 image2 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ shift 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size=128 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:13.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/ns1/image2 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ is_pool_mirror_mode_image mirror_parent/ns1 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' image = image ']' 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ enable_mirror cluster2 mirror_parent/ns1 image2 snapshot 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local mode=snapshot 2026-02-22T22:39:13.117 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/ns1/image2 snapshot 2026-02-22T22:39:13.585 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Mirroring enabled 2026-02-22T22:39:13.590 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster=cluster2 info mirror_parent/ns1/image2 2026-02-22T22:39:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:rbd image 'image2': 2026-02-22T22:39:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: size 128 MiB in 32 objects 2026-02-22T22:39:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: order 22 (4 MiB objects) 2026-02-22T22:39:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: snapshot_count: 1 2026-02-22T22:39:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: id: 10e1c8a4e07a 2026-02-22T22:39:13.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: block_name_prefix: rbd_data.10e1c8a4e07a 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: format: 2 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: op_features: 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: flags: 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: create_timestamp: Sun Feb 22 22:39:13 2026 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: access_timestamp: Sun Feb 22 22:39:13 2026 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: modify_timestamp: Sun Feb 22 22:39:13 2026 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: journal: 10e1c8a4e07a 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring state: enabled 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring mode: snapshot 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring global id: 622e1570-7340-40c3-942b-861b8ca645dc 2026-02-22T22:39:13.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: mirroring primary: true 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_started cluster2 mirror_parent image1 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_state cluster2 mirror_parent image1 started 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=started 2026-02-22T22:39:13.617 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local s 2026-02-22T22:39:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:13.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_image_replay_state cluster2 mirror_parent image1 started 2026-02-22T22:39:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:14.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local test_state=started 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_result 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local current_state=stopped 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemons cluster2 rbd mirror status mirror_parent/image1 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_instance=cluster2 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ grep -i state 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance=cluster2 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local loop_instance 2026-02-22T22:39:14.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:14.620 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sleep 0 2026-02-22T22:39:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:39:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ seq 0 0 2026-02-22T22:39:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror_parent/image1 2026-02-22T22:39:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2:0 2026-02-22T22:39:14.623 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster2:0 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster2 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:39:14.624 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster2 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster2:0 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:39:14.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ local_cluster=cluster2 2026-02-22T22:39:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:39:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:39:14.626 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:39:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:14.633 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror_parent/image1 2026-02-22T22:39:14.680 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ return 0 2026-02-22T22:39:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:39:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:39:14.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep -i Replaying 2026-02-22T22:39:14.682 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: "state": "Replaying" 2026-02-22T22:39:14.683 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ current_state=started 2026-02-22T22:39:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test started = started 2026-02-22T22:39:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ write_image cluster1 mirror_parent image1 100 2026-02-22T22:39:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:14.684 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local count=100 2026-02-22T22:39:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size= 2026-02-22T22:39:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:39:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ size=4096 2026-02-22T22:39:14.685 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 bench mirror_parent/image1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:39:14.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:39:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:39:14.731 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:elapsed: 0 ops: 100 ops/sec: 7143.12 bytes/sec: 28 MiB/s 2026-02-22T22:39:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror_parent image1 2026-02-22T22:39:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster2 2026-02-22T22:39:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:14.738 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:39:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ mirror_image_snapshot cluster1 mirror_parent image1 2026-02-22T22:39:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:14.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror_parent/image1 2026-02-22T22:39:15.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Snapshot ID: 6 2026-02-22T22:39:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror_parent image1 2026-02-22T22:39:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster2 2026-02-22T22:39:15.062 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster1-mirror_parent-image1.status 2026-02-22T22:39:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster1-mirror_parent-image1.status 2026-02-22T22:39:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:15.063 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent-image1.status 2026-02-22T22:39:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster2-mirror_parent-image1.status 2026-02-22T22:39:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster2-mirror_parent-image1.status 2026-02-22T22:39:15.064 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:15.065 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_status_log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent-image1.status 2026-02-22T22:39:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent image1 /tmp/tmp.AySj3raLFU/cluster1-mirror_parent-image1.status 2026-02-22T22:39:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent-image1.status 2026-02-22T22:39:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/image1 --format xml 2026-02-22T22:39:15.066 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:15.094 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:39:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local snapshot_id=6 2026-02-22T22:39:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ true 2026-02-22T22:39:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:15.096 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.2 2026-02-22T22:39:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent image1 /tmp/tmp.AySj3raLFU/cluster2-mirror_parent-image1.status 2026-02-22T22:39:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent-image1.status 2026-02-22T22:39:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/image1 --format xml 2026-02-22T22:39:15.298 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:15.326 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=3 2026-02-22T22:39:15.327 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 6 = 3 2026-02-22T22:39:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:15.328 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.4 2026-02-22T22:39:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent image1 /tmp/tmp.AySj3raLFU/cluster2-mirror_parent-image1.status 2026-02-22T22:39:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent-image1.status 2026-02-22T22:39:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/image1 --format xml 2026-02-22T22:39:15.728 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:15.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:15.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=6 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 6 = 6 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror_parent image1 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_status_in_pool_dir cluster2 mirror_parent image1 up+replaying 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:15.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:15.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_status_in_pool_dir cluster2 mirror_parent image1 up+replaying '' '' 2026-02-22T22:39:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:16.760 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:16.761 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror image status mirror_parent/image1 2026-02-22T22:39:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='image1: 2026-02-22T22:39:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: global_id: b62f7034-d36e-473d-8201-0c6d09c052f8 2026-02-22T22:39:16.794 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+replaying 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799952,"replay_state":"idle"} 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: service: mirror.0 on trial178 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:14 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: peer_sites: 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: name: cluster1 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: down+unknown 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: status not found 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: ' 2026-02-22T22:39:16.795 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:39:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: state: up+replaying 2026-02-22T22:39:16.796 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ description: .*' 2026-02-22T22:39:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799952,"replay_state":"idle"} 2026-02-22T22:39:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:16.797 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^up+' 2026-02-22T22:39:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:39:16.798 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: service: mirror.0 on trial178 2026-02-22T22:39:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update 2026-02-22T22:39:16.799 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:39:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:14' 2026-02-22T22:39:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_mirror_pool_status_verbose cluster2 mirror_parent image1 up+replaying '2026-02-22 22:39:14' 2026-02-22T22:39:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:16.800 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:16.801 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:16.801 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local 'prev_last_update=2026-02-22 22:39:14' 2026-02-22T22:39:16.801 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:16.801 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:16.801 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format xml 2026-02-22T22:39:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='WARNINGOKWARNING1142654276mirror.0trial17819.2.3-689-gf90f4eaetrueOKimage1b62f7034-d36e-473d-8201-0c6d09c052f8up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799952,"replay_state":"idle"}42654276mirror.0trial1782026-02-22 22:39:14cluster1dec1549c-b39d-4128-91e3-03c121098ba9down+unknownstatus not foundimage2cd82a87a-1ebd-4f76-98ef-2422e809f845up+stoppedlocal image is primary42654276mirror.0trial1782026-02-22 22:39:14cluster1dec1549c-b39d-4128-91e3-03c121098ba9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799952,"replay_state":"idle"}2026-02-22 22:39:14' 2026-02-22T22:39:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update state 2026-02-22T22:39:16.831 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/last_update' 2026-02-22T22:39:16.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:14' 2026-02-22T22:39:16.833 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/state' 2026-02-22T22:39:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ state=up+replaying 2026-02-22T22:39:16.834 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep up+replaying 2026-02-22T22:39:16.835 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_started cluster1 mirror_parent image2 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_state cluster1 mirror_parent image2 started 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:16.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=started 2026-02-22T22:39:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local s 2026-02-22T22:39:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:16.838 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_image_replay_state cluster1 mirror_parent image2 started 2026-02-22T22:39:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:17.836 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local test_state=started 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_result 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local current_state=stopped 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/image2 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ grep -i state 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_instance=cluster1 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance=cluster1 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local loop_instance 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:17.837 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sleep 0 2026-02-22T22:39:17.838 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:39:17.839 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ seq 0 0 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/image2 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1:0 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster1:0 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster1 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster1 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster1 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:39:17.840 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster1 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster1:0 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster1 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ local_cluster=cluster1 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:39:17.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:39:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:39:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:39:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:39:17.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/image2 2026-02-22T22:39:17.900 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ return 0 2026-02-22T22:39:17.901 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:39:17.901 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:39:17.901 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep -i Replaying 2026-02-22T22:39:17.902 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: "state": "Replaying" 2026-02-22T22:39:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ current_state=started 2026-02-22T22:39:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test started = started 2026-02-22T22:39:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ write_image cluster2 mirror_parent image2 100 2026-02-22T22:39:17.903 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local count=100 2026-02-22T22:39:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size= 2026-02-22T22:39:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:39:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ size=4096 2026-02-22T22:39:17.904 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 bench mirror_parent/image2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:39:17.936 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:39:17.950 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:39:17.950 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:elapsed: 0 ops: 100 ops/sec: 7143.12 bytes/sec: 28 MiB/s 2026-02-22T22:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent image2 2026-02-22T22:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster1 2026-02-22T22:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:17.957 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:39:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ mirror_image_snapshot cluster2 mirror_parent image2 2026-02-22T22:39:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:17.958 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/image2 2026-02-22T22:39:18.599 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Snapshot ID: 8 2026-02-22T22:39:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror_parent image2 2026-02-22T22:39:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster1 2026-02-22T22:39:18.605 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:18.606 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:18.606 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:18.606 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster2-mirror_parent-image2.status 2026-02-22T22:39:18.606 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster2-mirror_parent-image2.status 2026-02-22T22:39:18.606 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:18.607 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent-image2.status 2026-02-22T22:39:18.607 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster1-mirror_parent-image2.status 2026-02-22T22:39:18.608 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster1-mirror_parent-image2.status 2026-02-22T22:39:18.608 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_status_log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent-image2.status 2026-02-22T22:39:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent image2 /tmp/tmp.AySj3raLFU/cluster2-mirror_parent-image2.status 2026-02-22T22:39:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:18.609 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:18.610 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:18.610 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent-image2.status 2026-02-22T22:39:18.610 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/image2 --format xml 2026-02-22T22:39:18.610 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:18.637 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:39:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local snapshot_id=8 2026-02-22T22:39:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ true 2026-02-22T22:39:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:18.639 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.2 2026-02-22T22:39:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent image2 /tmp/tmp.AySj3raLFU/cluster1-mirror_parent-image2.status 2026-02-22T22:39:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent-image2.status 2026-02-22T22:39:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/image2 --format xml 2026-02-22T22:39:18.841 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:18.870 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=3 2026-02-22T22:39:18.871 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 8 = 3 2026-02-22T22:39:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:18.872 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.4 2026-02-22T22:39:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent image2 /tmp/tmp.AySj3raLFU/cluster1-mirror_parent-image2.status 2026-02-22T22:39:19.273 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent-image2.status 2026-02-22T22:39:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/image2 --format xml 2026-02-22T22:39:19.274 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:19.304 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=8 2026-02-22T22:39:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 8 = 8 2026-02-22T22:39:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent image2 2026-02-22T22:39:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:19.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent image2 up+replaying 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:19.308 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:20.306 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_status_in_pool_dir cluster1 mirror_parent image2 up+replaying '' '' 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:20.307 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/image2 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='image2: 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: global_id: cd82a87a-1ebd-4f76-98ef-2422e809f845 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+replaying 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799952,"replay_state":"idle"} 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: service: mirror.0 on trial178 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:14 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: peer_sites: 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: name: cluster2 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+stopped 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: local image is primary 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:17' 2026-02-22T22:39:20.339 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:39:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: state: up+replaying 2026-02-22T22:39:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ description: .*' 2026-02-22T22:39:20.340 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799952,"replay_state":"idle"} 2026-02-22T22:39:20.341 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:20.341 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:20.341 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^up+' 2026-02-22T22:39:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:20.342 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:39:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: service: mirror.0 on trial178 2026-02-22T22:39:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update 2026-02-22T22:39:20.343 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:39:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:14' 2026-02-22T22:39:20.345 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent image2 up+replaying '2026-02-22 22:39:14' 2026-02-22T22:39:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local 'prev_last_update=2026-02-22 22:39:14' 2026-02-22T22:39:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:20.346 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format xml 2026-02-22T22:39:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='OKOKOK242684271mirror.0trial17819.2.3-689-gf90f4eaetrueOKimage1b62f7034-d36e-473d-8201-0c6d09c052f8up+stoppedlocal image is primary42684271mirror.0trial1782026-02-22 22:39:17cluster27670aebc-b99e-4341-b811-faecd2cff159up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799952,"replay_state":"idle"}2026-02-22 22:39:14image2cd82a87a-1ebd-4f76-98ef-2422e809f845up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799952,"replay_state":"idle"}42684271mirror.0trial1782026-02-22 22:39:14cluster27670aebc-b99e-4341-b811-faecd2cff159up+stoppedlocal image is primary2026-02-22 22:39:17' 2026-02-22T22:39:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update state 2026-02-22T22:39:20.378 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image2'\'']/last_update' 2026-02-22T22:39:20.379 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:14' 2026-02-22T22:39:20.380 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image2'\'']/state' 2026-02-22T22:39:20.381 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ state=up+replaying 2026-02-22T22:39:20.381 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:20.382 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep up+replaying 2026-02-22T22:39:20.382 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:20.383 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:20.383 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:20.383 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster1 mirror_parent 2026-02-22T22:39:20.383 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:20.383 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:20.383 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:20.384 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4268","instance_id":"4271","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:17","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:14"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:14","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:17"}]}]}' 2026-02-22T22:39:20.418 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.states == {"replaying": 2}' 2026-02-22T22:39:20.419 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:20.420 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:14","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:17"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:17","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:14"}]}]}' 2026-02-22T22:39:20.450 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.states == {"replaying": 2}' 2026-02-22T22:39:20.451 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_started cluster2 mirror_parent/ns1 image1 2026-02-22T22:39:20.452 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_state cluster2 mirror_parent/ns1 image1 started 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=started 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local s 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:20.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_image_replay_state cluster2 mirror_parent/ns1 image1 started 2026-02-22T22:39:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local test_state=started 2026-02-22T22:39:21.453 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_result 2026-02-22T22:39:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local current_state=stopped 2026-02-22T22:39:21.454 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemons cluster2 rbd mirror status mirror_parent/ns1/image1 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_instance=cluster2 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance=cluster2 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local loop_instance 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sleep 0 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ grep -i state 2026-02-22T22:39:21.455 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:39:21.456 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ seq 0 0 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror_parent/ns1/image1 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2:0 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster2:0 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster2 2026-02-22T22:39:21.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster2 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster2:0 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:39:21.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ local_cluster=cluster2 2026-02-22T22:39:21.460 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:39:21.460 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:39:21.460 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:39:21.470 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:21.470 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:21.470 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:39:21.470 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror_parent/ns1/image1 2026-02-22T22:39:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ return 0 2026-02-22T22:39:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:39:21.518 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:39:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep -i Replaying 2026-02-22T22:39:21.519 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: "state": "Replaying" 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ current_state=started 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test started = started 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ write_image cluster1 mirror_parent/ns1 image1 100 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local count=100 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size= 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ size=4096 2026-02-22T22:39:21.520 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 bench mirror_parent/ns1/image1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:39:21.557 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:39:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:39:22.062 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:elapsed: 0 ops: 100 ops/sec: 198.42 bytes/sec: 794 KiB/s 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror_parent/ns1 image1 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster2 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ mirror_image_snapshot cluster1 mirror_parent/ns1 image1 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:22.071 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:22.072 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:22.072 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 mirror image snapshot mirror_parent/ns1/image1 2026-02-22T22:39:23.072 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Snapshot ID: 9 2026-02-22T22:39:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_snapshot_sync_complete cluster2 cluster1 mirror_parent/ns1 image1 2026-02-22T22:39:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster2 2026-02-22T22:39:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:23.078 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster1-mirror_parent/ns1-image1.status 2026-02-22T22:39:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster1-mirror_parent/ns1-image1.status 2026-02-22T22:39:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:23.079 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent_ns1-image1.status 2026-02-22T22:39:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:39:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:39:23.080 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:23.081 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_status_log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:39:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent/ns1 image1 /tmp/tmp.AySj3raLFU/cluster1-mirror_parent_ns1-image1.status 2026-02-22T22:39:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent_ns1-image1.status 2026-02-22T22:39:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/ns1/image1 --format xml 2026-02-22T22:39:23.082 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:23.110 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:39:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local snapshot_id=9 2026-02-22T22:39:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ true 2026-02-22T22:39:23.112 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:23.113 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.2 2026-02-22T22:39:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent/ns1 image1 /tmp/tmp.AySj3raLFU/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:39:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:23.313 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:39:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/ns1/image1 --format xml 2026-02-22T22:39:23.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:23.341 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=4 2026-02-22T22:39:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 9 = 4 2026-02-22T22:39:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:23.343 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.4 2026-02-22T22:39:23.744 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent/ns1 image1 /tmp/tmp.AySj3raLFU/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:39:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:39:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/ns1/image1 --format xml 2026-02-22T22:39:23.745 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:23.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=9 2026-02-22T22:39:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 9 = 9 2026-02-22T22:39:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror_parent/ns1 image1 2026-02-22T22:39:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:23.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_status_in_pool_dir cluster2 mirror_parent/ns1 image1 up+replaying 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:23.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_status_in_pool_dir cluster2 mirror_parent/ns1 image1 up+replaying '' '' 2026-02-22T22:39:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:24.777 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:24.778 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror image status mirror_parent/ns1/image1 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='image1: 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: global_id: c1154afb-c3df-43a9-8813-a4d98bf6cbfc 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+replaying 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799953,"replay_state":"idle"} 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: service: mirror.0 on trial178 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:15 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: peer_sites: 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: name: cluster1 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+stopped 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: local image is primary 2026-02-22T22:39:24.809 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:22' 2026-02-22T22:39:24.810 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:39:24.810 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: state: up+replaying 2026-02-22T22:39:24.810 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ description: .*' 2026-02-22T22:39:24.811 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799953,"replay_state":"idle"} 2026-02-22T22:39:24.811 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:24.811 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:24.811 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^up+' 2026-02-22T22:39:24.812 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:24.812 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:39:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: service: mirror.0 on trial178 2026-02-22T22:39:24.813 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update 2026-02-22T22:39:24.814 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:15' 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_mirror_pool_status_verbose cluster2 mirror_parent/ns1 image1 up+replaying '2026-02-22 22:39:15' 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image1 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local 'prev_last_update=2026-02-22 22:39:15' 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:24.815 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:24.816 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent/ns1 --verbose --format xml 2026-02-22T22:39:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='OKOKOK242654276mirror.0trial17819.2.3-689-gf90f4eaetrueOKimage1c1154afb-c3df-43a9-8813-a4d98bf6cbfcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799953,"replay_state":"idle"}42654276mirror.0trial1782026-02-22 22:39:15cluster1dec1549c-b39d-4128-91e3-03c121098ba9up+stoppedlocal image is primary2026-02-22 22:39:22image2622e1570-7340-40c3-942b-861b8ca645dcup+stoppedlocal image is primary42654276mirror.0trial1782026-02-22 22:39:22cluster1dec1549c-b39d-4128-91e3-03c121098ba9up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799953,"replay_state":"idle"}2026-02-22 22:39:15' 2026-02-22T22:39:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update state 2026-02-22T22:39:24.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/last_update' 2026-02-22T22:39:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:15' 2026-02-22T22:39:24.848 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/state' 2026-02-22T22:39:24.850 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ state=up+replaying 2026-02-22T22:39:24.850 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:24.850 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep up+replaying 2026-02-22T22:39:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:24.851 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_started cluster1 mirror_parent/ns1 image2 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_image_replay_state cluster1 mirror_parent/ns1 image2 started 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=started 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local s 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:24.852 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:25.853 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_image_replay_state cluster1 mirror_parent/ns1 image2 started 2026-02-22T22:39:25.853 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:25.853 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:25.853 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:25.853 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local test_state=started 2026-02-22T22:39:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_result 2026-02-22T22:39:25.854 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local current_state=stopped 2026-02-22T22:39:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/ns1/image2 2026-02-22T22:39:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_instance=cluster1 2026-02-22T22:39:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance=cluster1 2026-02-22T22:39:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local loop_instance 2026-02-22T22:39:25.855 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:25.856 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ grep -i state 2026-02-22T22:39:25.856 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sleep 0 2026-02-22T22:39:25.856 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:39:25.856 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ seq 0 0 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/ns1/image2 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1:0 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ shift 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster1:0 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster1 2026-02-22T22:39:25.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster1 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster1 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster1 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster1:0 2026-02-22T22:39:25.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster1 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ local_cluster=cluster1 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:39:25.861 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:39:25.870 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:39:25.871 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:39:25.871 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:39:25.871 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/ns1/image2 2026-02-22T22:39:25.918 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ return 0 2026-02-22T22:39:25.918 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:39:25.919 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:39:25.919 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep -i Replaying 2026-02-22T22:39:25.920 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: "state": "Replaying" 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ current_state=started 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test started = started 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ write_image cluster2 mirror_parent/ns1 image2 100 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local count=100 2026-02-22T22:39:25.921 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local size= 2026-02-22T22:39:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:39:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ size=4096 2026-02-22T22:39:25.922 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 bench mirror_parent/ns1/image2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:39:25.954 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:39:25.966 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:39:25.966 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:elapsed: 0 ops: 100 ops/sec: 9091.15 bytes/sec: 36 MiB/s 2026-02-22T22:39:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent/ns1 image2 2026-02-22T22:39:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster1 2026-02-22T22:39:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:25.973 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = snapshot ']' 2026-02-22T22:39:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ mirror_image_snapshot cluster2 mirror_parent/ns1 image2 2026-02-22T22:39:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:25.974 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 mirror image snapshot mirror_parent/ns1/image2 2026-02-22T22:39:26.609 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:Snapshot ID: 10 2026-02-22T22:39:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_snapshot_sync_complete cluster1 cluster2 mirror_parent/ns1 image2 2026-02-22T22:39:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_cluster=cluster1 2026-02-22T22:39:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:26.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:39:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:39:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:26.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status_log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:39:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ mkfname cluster1-mirror_parent/ns1-image2.status 2026-02-22T22:39:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo cluster1-mirror_parent/ns1-image2.status 2026-02-22T22:39:26.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -e 's|[/ ]|_|g' 2026-02-22T22:39:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local local_status_log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent_ns1-image2.status 2026-02-22T22:39:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster2 mirror_parent/ns1 image2 /tmp/tmp.AySj3raLFU/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:39:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:26.618 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:39:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 snap list --all mirror_parent/ns1/image2 --format xml 2026-02-22T22:39:26.619 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:26.646 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/id 2026-02-22T22:39:26.647 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local snapshot_id=10 2026-02-22T22:39:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ true 2026-02-22T22:39:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:26.648 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.2 2026-02-22T22:39:26.849 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent/ns1 image2 /tmp/tmp.AySj3raLFU/cluster1-mirror_parent_ns1-image2.status 2026-02-22T22:39:26.849 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:26.849 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:26.849 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:26.849 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent_ns1-image2.status 2026-02-22T22:39:26.849 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/ns1/image2 --format xml 2026-02-22T22:39:26.850 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:26.880 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:26.881 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=4 2026-02-22T22:39:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 10 = 4 2026-02-22T22:39:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:39:26.882 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0.4 2026-02-22T22:39:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ get_newest_mirror_snapshot cluster1 mirror_parent/ns1 image2 /tmp/tmp.AySj3raLFU/cluster1-mirror_parent_ns1-image2.status 2026-02-22T22:39:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local log=/tmp/tmp.AySj3raLFU/cluster1-mirror_parent_ns1-image2.status 2026-02-22T22:39:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster1 snap list --all mirror_parent/ns1/image2 --format xml 2026-02-22T22:39:27.283 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ xmlstarlet sel -t -c '//snapshots/snapshot[namespace/complete='\''true'\'' and position()=last()]' 2026-02-22T22:39:27.311 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v //snapshot/namespace/primary_snap_id 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local primary_snapshot_id=10 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test 10 = 10 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent/ns1 image2 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' snapshot = journal ']' 2026-02-22T22:39:27.314 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent/ns1 image2 up+replaying 2026-02-22T22:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:27.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_status_in_pool_dir cluster1 mirror_parent/ns1 image2 up+replaying '' '' 2026-02-22T22:39:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:28.315 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local description_pattern= 2026-02-22T22:39:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local service_pattern= 2026-02-22T22:39:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:28.316 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/ns1/image2 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='image2: 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: global_id: 622e1570-7340-40c3-942b-861b8ca645dc 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+replaying 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799953,"replay_state":"idle"} 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: service: mirror.0 on trial178 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:15 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: peer_sites: 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: name: cluster2 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: state: up+stopped 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: description: local image is primary 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr: last_update: 2026-02-22 22:39:27' 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:39:28.347 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: state: up+replaying 2026-02-22T22:39:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^ description: .*' 2026-02-22T22:39:28.348 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: description: replaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799953,"replay_state":"idle"} 2026-02-22T22:39:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n '' ']' 2026-02-22T22:39:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:28.349 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep '^up+' 2026-02-22T22:39:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:28.350 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:39:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout: service: mirror.0 on trial178 2026-02-22T22:39:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update 2026-02-22T22:39:28.351 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:15' 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent/ns1 image2 up+replaying '2026-02-22 22:39:15' 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local image=image2 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state_pattern=up+replaying 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local 'prev_last_update=2026-02-22 22:39:15' 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:28.353 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent/ns1 --verbose --format xml 2026-02-22T22:39:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='OKOKOK242684271mirror.0trial17819.2.3-689-gf90f4eaetrueOKimage1c1154afb-c3df-43a9-8813-a4d98bf6cbfcup+stoppedlocal image is primary42684271mirror.0trial1782026-02-22 22:39:27cluster27670aebc-b99e-4341-b811-faecd2cff159up+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799953,"replay_state":"idle"}2026-02-22 22:39:15image2622e1570-7340-40c3-942b-861b8ca645dcup+replayingreplaying, {"bytes_per_second":0.0,"bytes_per_snapshot":0.0,"last_snapshot_bytes":0,"last_snapshot_sync_seconds":0,"remote_snapshot_timestamp":1771799953,"replay_state":"idle"}42684271mirror.0trial1782026-02-22 22:39:15cluster27670aebc-b99e-4341-b811-faecd2cff159up+stoppedlocal image is primary2026-02-22 22:39:27' 2026-02-22T22:39:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local last_update state 2026-02-22T22:39:28.384 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image2'\'']/last_update' 2026-02-22T22:39:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ last_update='2026-02-22 22:39:15' 2026-02-22T22:39:28.386 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image2'\'']/state' 2026-02-22T22:39:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ state=up+replaying 2026-02-22T22:39:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo up+replaying 2026-02-22T22:39:28.388 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ grep up+replaying 2026-02-22T22:39:28.389 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:up+replaying 2026-02-22T22:39:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster1 mirror_parent/ns1 2026-02-22T22:39:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:39:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:28.390 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent/ns1 --verbose --format json 2026-02-22T22:39:28.421 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4268","instance_id":"4271","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"c1154afb-c3df-43a9-8813-a4d98bf6cbfc","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:27","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799953,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:15"}]},{"name":"image2","global_id":"622e1570-7340-40c3-942b-861b8ca645dc","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799953,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:15","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:27"}]}]}' 2026-02-22T22:39:28.421 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.states == {"replaying": 2}' 2026-02-22T22:39:28.422 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent/ns1 2026-02-22T22:39:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:39:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:28.423 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent/ns1 --verbose --format json 2026-02-22T22:39:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"c1154afb-c3df-43a9-8813-a4d98bf6cbfc","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799953,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:27"}]},{"name":"image2","global_id":"622e1570-7340-40c3-942b-861b8ca645dc","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:27","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799953,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:15"}]}]}' 2026-02-22T22:39:28.454 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.states == {"replaying": 2}' 2026-02-22T22:39:28.456 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ testlog 'TEST: pool replayer and callout cleanup when peer is updated' 2026-02-22T22:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ tee -a /tmp/tmp.AySj3raLFU/rbd-mirror.test.log 2026-02-22T22:39:28.457 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ date '+%F %T' 2026-02-22T22:39:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo 2026-02-22 22:39:28 TEST: pool replayer and callout cleanup when peer is updated 2026-02-22T22:39:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:2026-02-22 22:39:28 TEST: pool replayer and callout cleanup when peer is updated 2026-02-22T22:39:28.458 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster1 mirror_parent OK 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=OK 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster1 mirror_parent 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:28.459 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4268","instance_id":"4271","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:27","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:14"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:14","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:27"}]}]}' 2026-02-22T22:39:28.490 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:28.491 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:39:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:28.492 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=OK 2026-02-22T22:39:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:28.493 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:28.522 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:14","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:27"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:27","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:14"}]}]}' 2026-02-22T22:39:28.522 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:28.523 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:28.524 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:28.554 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:14","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:27"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:27","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:14"}]}]}' 2026-02-22T22:39:28.555 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:28.556 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.daemon_health == "OK"' 2026-02-22T22:39:28.557 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:28.558 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.daemons[0].health == "OK"' 2026-02-22T22:39:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:28.559 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.daemons[0] | has("callouts") | not' 2026-02-22T22:39:28.560 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:28.561 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ jq -r '.daemons[0].service_id' 2026-02-22T22:39:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ OLD_SERVICE_ID=4265 2026-02-22T22:39:28.563 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ jq -r '.daemons[0].instance_id' 2026-02-22T22:39:28.565 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ OLD_INSTANCE_ID=4276 2026-02-22T22:39:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 --pool mirror_parent mirror pool info --format json 2026-02-22T22:39:28.566 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ jq -r '.peers[0].uuid' 2026-02-22T22:39:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ PEER_UUID=795f72f5-603c-4a54-8a29-cfe75d2fdd66 2026-02-22T22:39:28.590 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 --pool mirror_parent mirror pool peer set 795f72f5-603c-4a54-8a29-cfe75d2fdd66 client client.invalid 2026-02-22T22:39:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_health_state cluster2 mirror_parent ERROR 2026-02-22T22:39:28.614 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=ERROR 2026-02-22T22:39:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local s 2026-02-22T22:39:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:28.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent ERROR 2026-02-22T22:39:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:29.615 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=ERROR 2026-02-22T22:39:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:29.616 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:14","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:27"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:27","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:14"}]}]}' 2026-02-22T22:39:29.645 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:39:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:false 2026-02-22T22:39:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:29.646 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 2 2026-02-22T22:39:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent ERROR 2026-02-22T22:39:31.648 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=ERROR 2026-02-22T22:39:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:31.649 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:31.678 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:14","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:27"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4265","instance_id":"4276","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:27","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"remote_snapshot_timestamp\":1771799952,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:14"}]}]}' 2026-02-22T22:39:31.678 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:39:31.679 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:false 2026-02-22T22:39:31.679 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:31.679 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 4 2026-02-22T22:39:35.680 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent ERROR 2026-02-22T22:39:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=ERROR 2026-02-22T22:39:35.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:35.682 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:35.713 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:32"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:35.713 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:39:35.714 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:false 2026-02-22T22:39:35.715 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:35.715 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 8 2026-02-22T22:39:43.716 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent ERROR 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=ERROR 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:43.717 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:43.750 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:42"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:43.750 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:39:43.751 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster1 mirror_parent WARNING 2026-02-22T22:39:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:43.752 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=WARNING 2026-02-22T22:39:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster1 mirror_parent 2026-02-22T22:39:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:43.753 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:43.786 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"WARNING","daemon_health":"OK","image_health":"WARNING","states":{"unknown":2}},"daemons":[{"service_id":"4268","instance_id":"4271","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:42","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:39:32"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:43.787 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "WARNING"' 2026-02-22T22:39:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:43.788 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:43.789 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:42"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:43.818 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:39:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:43.820 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.daemon_health == "ERROR"' 2026-02-22T22:39:43.821 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:43.822 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.daemons[0].health == "ERROR"' 2026-02-22T22:39:43.823 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:43.824 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.daemons[0].callouts == ["unable to connect to remote cluster"]' 2026-02-22T22:39:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:43.825 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rbd --cluster cluster2 --pool mirror_parent mirror pool peer set 795f72f5-603c-4a54-8a29-cfe75d2fdd66 client client.rbd-mirror-peer 2026-02-22T22:39:44.054 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ wait_for_health_state cluster2 mirror_parent OK 2026-02-22T22:39:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=OK 2026-02-22T22:39:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local s 2026-02-22T22:39:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:44.055 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:39:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:39:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=OK 2026-02-22T22:39:45.056 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:45.057 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:42"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:45.087 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:false 2026-02-22T22:39:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:45.089 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 2 2026-02-22T22:39:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:39:47.090 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=OK 2026-02-22T22:39:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:47.091 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:47.121 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:47"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:47.121 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:false 2026-02-22T22:39:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:47.122 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 4 2026-02-22T22:39:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:39:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=OK 2026-02-22T22:39:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:51.124 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:51.125 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4265","instance_id":"4276","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"down+unknown","description":"status not found","last_update":"","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:47"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"down+unknown","description":"status not found","last_update":"","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:51.154 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:false 2026-02-22T22:39:51.155 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:39:51.156 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 8 2026-02-22T22:39:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:39:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:39:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=OK 2026-02-22T22:39:59.157 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:59.158 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4265","instance_id":"4423","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799955,\"remote_snapshot_timestamp\":1771799955,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4423","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:51","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:57"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4265","instance_id":"4423","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:57","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:59.188 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:59.189 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:59.190 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ return 0 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test_health_state cluster1 mirror_parent OK 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pool=mirror_parent 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local state=OK 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local status 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster1 mirror_parent 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:59.191 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4268","instance_id":"4271","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:57","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799955,\"remote_snapshot_timestamp\":1771799955,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:51"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4268","instance_id":"4271","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:32","peer_sites":[{"site_name":"cluster2","mirror_uuids":"7670aebc-b99e-4341-b811-faecd2cff159","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:57"}]}]}' 2026-02-22T22:39:59.222 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:59.224 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:39:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:39:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local pool=mirror_parent 2026-02-22T22:39:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ CEPH_ARGS= 2026-02-22T22:39:59.225 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:39:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4265","instance_id":"4423","client_id":"mirror.0","hostname":"trial178","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"b62f7034-d36e-473d-8201-0c6d09c052f8","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"bytes_per_snapshot\":0.0,\"last_snapshot_bytes\":0,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799955,\"remote_snapshot_timestamp\":1771799955,\"replay_state\":\"idle\"}","daemon_service":{"service_id":"4265","instance_id":"4423","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:51","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:39:57"}]},{"name":"image2","global_id":"cd82a87a-1ebd-4f76-98ef-2422e809f845","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4265","instance_id":"4423","daemon_id":"mirror.0","hostname":"trial178"},"last_update":"2026-02-22 22:39:57","peer_sites":[{"site_name":"cluster1","mirror_uuids":"dec1549c-b39d-4128-91e3-03c121098ba9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":40960.0,\"bytes_per_snapshot\":204800.0,\"last_snapshot_bytes\":409600,\"last_snapshot_sync_seconds\":0,\"local_snapshot_timestamp\":1771799958,\"remote_snapshot_timestamp\":1771799958,\"replay_state\":\"idle\"}","last_update":"2026-02-22 22:39:32"}]}]}' 2026-02-22T22:39:59.254 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:39:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:59.256 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.summary.daemon_health == "OK"' 2026-02-22T22:39:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:59.257 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.daemons[0].health == "OK"' 2026-02-22T22:39:59.259 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:59.259 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ jq -e '.daemons[0] | has("callouts") | not' 2026-02-22T22:39:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:true 2026-02-22T22:39:59.261 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ jq -r '.daemons[0].service_id' 2026-02-22T22:39:59.263 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ NEW_SERVICE_ID=4265 2026-02-22T22:39:59.263 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ jq -r '.daemons[0].instance_id' 2026-02-22T22:39:59.266 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ NEW_INSTANCE_ID=4423 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ (( OLD_SERVICE_ID == NEW_SERVICE_ID )) 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ (( OLD_INSTANCE_ID < NEW_INSTANCE_ID )) 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ cleanup 0 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local error_code=0 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ set +e 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' 0 -ne 0 ']' 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -z '' ']' 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster instance 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:39:59.267 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:39:59.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:pool 'mirror' does not exist 2026-02-22T22:39:59.735 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:39:59.736 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:40:00.625 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:pool 'mirror' does not exist 2026-02-22T22:40:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:40:00.635 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:40:01.731 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:40:01.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ CEPH_ARGS= 2026-02-22T22:40:01.739 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:40:02.627 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:40:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:40:02.636 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ stop_mirrors cluster1 2026-02-22T22:40:02.637 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1 2026-02-22T22:40:02.637 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local sig= 2026-02-22T22:40:02.637 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ seq 0 0 2026-02-22T22:40:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:40:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:40:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster1:0 2026-02-22T22:40:02.639 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local sig= 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pid 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster1:0 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster1:0 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster1 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:40:02.640 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:40:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval cluster=cluster1 2026-02-22T22:40:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ cluster=cluster1 2026-02-22T22:40:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:40:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:40:02.641 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:40:02.650 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:40:02.651 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:40:02.652 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ pid=49160 2026-02-22T22:40:02.652 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n 49160 ']' 2026-02-22T22:40:02.652 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ kill 49160 2026-02-22T22:40:02.652 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:40:02.652 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0 2026-02-22T22:40:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ps auxww 2026-02-22T22:40:02.653 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ awk -v pid=49160 '$2 == pid {print; exit 1}' 2026-02-22T22:40:02.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:ubuntu 49160 0.2 0.0 523388 50648 ? Ssl 22:39 0:00 rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:40:02.681 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:40:02.682 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 1 2026-02-22T22:40:03.682 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ps auxww 2026-02-22T22:40:03.683 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ awk -v pid=49160 '$2 == pid {print; exit 1}' 2026-02-22T22:40:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ break 2026-02-22T22:40:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ps auxww 2026-02-22T22:40:03.706 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ awk -v pid=49160 '$2 == pid {print; exit 1}' 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster1:0 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster1 2026-02-22T22:40:03.729 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:40:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:40:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval local_cluster=cluster1 2026-02-22T22:40:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local_cluster=cluster1 2026-02-22T22:40:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:40:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:40:03.730 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:40:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:40:03.741 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:40:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:40:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:40:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:40:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:40:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:40:03.742 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster1:0 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster1 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval local_cluster=cluster1 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local_cluster=cluster1 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:40:03.743 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:40:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:40:03.756 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:40:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:40:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1:0 2026-02-22T22:40:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:40:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:40:03.758 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster1:0 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster1 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster1 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster1 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:40:03.759 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:40:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:40:03.769 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:40:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:40:03.770 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ stop_mirrors cluster2 2026-02-22T22:40:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2 2026-02-22T22:40:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local sig= 2026-02-22T22:40:03.771 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ seq 0 0 2026-02-22T22:40:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:40:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T22:40:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local cluster=cluster2:0 2026-02-22T22:40:03.773 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local sig= 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test -n '' 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ local pid 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster=cluster2:0 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local val=cluster2:0 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local instance_var_name=instance 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:40:03.774 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:40:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval cluster=cluster2 2026-02-22T22:40:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ cluster=cluster2 2026-02-22T22:40:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ eval instance=0 2026-02-22T22:40:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ instance=0 2026-02-22T22:40:03.775 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:40:03.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:40:03.783 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:40:03.784 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ pid=49205 2026-02-22T22:40:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -n 49205 ']' 2026-02-22T22:40:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ kill 49205 2026-02-22T22:40:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:40:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ sleep 0 2026-02-22T22:40:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ps auxww 2026-02-22T22:40:03.785 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ awk -v pid=49205 '$2 == pid {print; exit 1}' 2026-02-22T22:40:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ break 2026-02-22T22:40:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ ps auxww 2026-02-22T22:40:03.808 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ awk -v pid=49205 '$2 == pid {print; exit 1}' 2026-02-22T22:40:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T22:40:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:40:03.831 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster1 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster2:0 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval local_cluster=cluster2 2026-02-22T22:40:03.832 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local_cluster=cluster2 2026-02-22T22:40:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:40:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:40:03.833 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:40:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:40:03.842 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:40:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:40:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:40:03.844 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster2:0 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval local_cluster=cluster2 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ local_cluster=cluster2 2026-02-22T22:40:03.845 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:40:03.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:40:03.846 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:40:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:40:03.857 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:40:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ daemon_pid_file cluster2:0 2026-02-22T22:40:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster=cluster2:0 2026-02-22T22:40:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance 2026-02-22T22:40:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:40:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local val=cluster2:0 2026-02-22T22:40:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local cluster_var_name=cluster 2026-02-22T22:40:03.859 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ local instance_var_name=instance 2026-02-22T22:40:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ cluster=cluster2 2026-02-22T22:40:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ instance=0 2026-02-22T22:40:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:40:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval cluster=cluster2 2026-02-22T22:40:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ cluster=cluster2 2026-02-22T22:40:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ eval instance=0 2026-02-22T22:40:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ instance=0 2026-02-22T22:40:03.860 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:40:03.871 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:40:03.872 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' -z 1 ']' 2026-02-22T22:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ test '' = /tmp/tmp.AySj3raLFU 2026-02-22T22:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ rm -Rf /tmp/tmp.AySj3raLFU 2026-02-22T22:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:40:03.873 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ echo OK 2026-02-22T22:40:03.874 INFO:tasks.workunit.cluster1.client.mirror.trial178.stdout:OK 2026-02-22T22:40:03.875 INFO:tasks.workunit.cluster1.client.mirror.trial178.stderr:+ exit 0 2026-02-22T22:40:03.875 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-22T22:40:03.876 DEBUG:teuthology.orchestra.run.trial178:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-22T22:40:03.942 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_bootstrap.sh'] on cluster1.client.mirror... 2026-02-22T22:40:03.942 DEBUG:teuthology.orchestra.run.trial178:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-22T22:40:04.280 DEBUG:teuthology.parallel:result is None 2026-02-22T22:40:04.281 DEBUG:teuthology.orchestra.run.trial178:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:40:04.306 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:40:04.306 DEBUG:teuthology.orchestra.run.trial178:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:40:04.363 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:40:04.363 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:40:04.421 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:40:04.421 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:40:04.576 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:40:04.576 INFO:teuthology.orchestra.run.trial178.stderr:dumped all 2026-02-22T22:40:04.584 INFO:teuthology.orchestra.run.trial178.stdout:{"pg_ready":true,"pg_map":{"version":100,"stamp":"2026-02-22T22:40:03.438915+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":89688,"kb_used_data":8808,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283025832,"statfs":{"total":289910292480,"available":289818451968,"internally_reserved":0,"allocated":9019392,"data_stored":8242139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-558846990,"num_objects":-244,"num_object_clones":0,"num_object_copies":-488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-244,"num_whiteouts":0,"num_read":-4546,"num_read_kb":-5342,"num_write":-1011,"num_write_kb":-3545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.000848"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607296+0000","last_change":"2026-02-22T22:38:17.577006+0000","last_active":"2026-02-22T22:39:34.607296+0000","last_peered":"2026-02-22T22:39:34.607296+0000","last_clean":"2026-02-22T22:39:34.607296+0000","last_became_active":"2026-02-22T22:38:15.574752+0000","last_became_peered":"2026-02-22T22:38:15.574752+0000","last_unstale":"2026-02-22T22:39:34.607296+0000","last_undegraded":"2026-02-22T22:39:34.607296+0000","last_fullsized":"2026-02-22T22:39:34.607296+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:05:37.350245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082003000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607286+0000","last_change":"2026-02-22T22:38:17.576762+0000","last_active":"2026-02-22T22:39:34.607286+0000","last_peered":"2026-02-22T22:39:34.607286+0000","last_clean":"2026-02-22T22:39:34.607286+0000","last_became_active":"2026-02-22T22:38:15.574306+0000","last_became_peered":"2026-02-22T22:38:15.574306+0000","last_unstale":"2026-02-22T22:39:34.607286+0000","last_undegraded":"2026-02-22T22:39:34.607286+0000","last_fullsized":"2026-02-22T22:39:34.607286+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:25:28.811738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059383000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607198+0000","last_change":"2026-02-22T22:38:17.576464+0000","last_active":"2026-02-22T22:39:34.607198+0000","last_peered":"2026-02-22T22:39:34.607198+0000","last_clean":"2026-02-22T22:39:34.607198+0000","last_became_active":"2026-02-22T22:38:15.574673+0000","last_became_peered":"2026-02-22T22:38:15.574673+0000","last_unstale":"2026-02-22T22:39:34.607198+0000","last_undegraded":"2026-02-22T22:39:34.607198+0000","last_fullsized":"2026-02-22T22:39:34.607198+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:46.550560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031834000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606534+0000","last_change":"2026-02-22T22:38:17.576467+0000","last_active":"2026-02-22T22:39:34.606534+0000","last_peered":"2026-02-22T22:39:34.606534+0000","last_clean":"2026-02-22T22:39:34.606534+0000","last_became_active":"2026-02-22T22:38:15.574763+0000","last_became_peered":"2026-02-22T22:38:15.574763+0000","last_unstale":"2026-02-22T22:39:34.606534+0000","last_undegraded":"2026-02-22T22:39:34.606534+0000","last_fullsized":"2026-02-22T22:39:34.606534+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:25.430048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031511400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":76,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.557968+0000","last_change":"2026-02-22T22:38:17.575058+0000","last_active":"2026-02-22T22:39:34.557968+0000","last_peered":"2026-02-22T22:39:34.557968+0000","last_clean":"2026-02-22T22:39:34.557968+0000","last_became_active":"2026-02-22T22:38:15.574409+0000","last_became_peered":"2026-02-22T22:38:15.574409+0000","last_unstale":"2026-02-22T22:39:34.557968+0000","last_undegraded":"2026-02-22T22:39:34.557968+0000","last_fullsized":"2026-02-22T22:39:34.557968+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:32:34.479960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053028299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606220+0000","last_change":"2026-02-22T22:38:17.677203+0000","last_active":"2026-02-22T22:39:34.606220+0000","last_peered":"2026-02-22T22:39:34.606220+0000","last_clean":"2026-02-22T22:39:34.606220+0000","last_became_active":"2026-02-22T22:38:15.574389+0000","last_became_peered":"2026-02-22T22:38:15.574389+0000","last_unstale":"2026-02-22T22:39:34.606220+0000","last_undegraded":"2026-02-22T22:39:34.606220+0000","last_fullsized":"2026-02-22T22:39:34.606220+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:09:24.494810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021430700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.605887+0000","last_change":"2026-02-22T22:38:17.677186+0000","last_active":"2026-02-22T22:39:34.605887+0000","last_peered":"2026-02-22T22:39:34.605887+0000","last_clean":"2026-02-22T22:39:34.605887+0000","last_became_active":"2026-02-22T22:38:15.574402+0000","last_became_peered":"2026-02-22T22:38:15.574402+0000","last_unstale":"2026-02-22T22:39:34.605887+0000","last_undegraded":"2026-02-22T22:39:34.605887+0000","last_fullsized":"2026-02-22T22:39:34.605887+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:19.472593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019295799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606727+0000","last_change":"2026-02-22T22:38:17.576831+0000","last_active":"2026-02-22T22:39:34.606727+0000","last_peered":"2026-02-22T22:39:34.606727+0000","last_clean":"2026-02-22T22:39:34.606727+0000","last_became_active":"2026-02-22T22:38:15.574263+0000","last_became_peered":"2026-02-22T22:38:15.574263+0000","last_unstale":"2026-02-22T22:39:34.606727+0000","last_undegraded":"2026-02-22T22:39:34.606727+0000","last_fullsized":"2026-02-22T22:39:34.606727+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:41.940013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068121299999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":165,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.605740+0000","last_change":"2026-02-22T22:38:11.502395+0000","last_active":"2026-02-22T22:39:34.605740+0000","last_peered":"2026-02-22T22:39:34.605740+0000","last_clean":"2026-02-22T22:39:34.605740+0000","last_became_active":"2026-02-22T22:38:11.501894+0000","last_became_peered":"2026-02-22T22:38:11.501894+0000","last_unstale":"2026-02-22T22:39:34.605740+0000","last_undegraded":"2026-02-22T22:39:34.605740+0000","last_fullsized":"2026-02-22T22:39:34.605740+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_clean_scrub_stamp":"2026-02-22T22:38:09.558114+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:19:00.206795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705689,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":29584,"kb_used_data":2624,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94342256,"statfs":{"total":96636764160,"available":96606470144,"internally_reserved":0,"allocated":2686976,"data_stored":2444772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738394,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":30168,"kb_used_data":3208,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94341672,"statfs":{"total":96636764160,"available":96605872128,"internally_reserved":0,"allocated":3284992,"data_stored":3014165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738394,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":29936,"kb_used_data":2976,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94341904,"statfs":{"total":96636764160,"available":96606109696,"internally_reserved":0,"allocated":3047424,"data_stored":2783202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:04.586 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:40:04.742 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:40:04.743 INFO:teuthology.orchestra.run.trial178.stderr:dumped all 2026-02-22T22:40:04.751 INFO:teuthology.orchestra.run.trial178.stdout:{"pg_ready":true,"pg_map":{"version":100,"stamp":"2026-02-22T22:40:03.438915+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":89688,"kb_used_data":8808,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283025832,"statfs":{"total":289910292480,"available":289818451968,"internally_reserved":0,"allocated":9019392,"data_stored":8242139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-558846990,"num_objects":-244,"num_object_clones":0,"num_object_copies":-488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-244,"num_whiteouts":0,"num_read":-4546,"num_read_kb":-5342,"num_write":-1011,"num_write_kb":-3545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.000848"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607296+0000","last_change":"2026-02-22T22:38:17.577006+0000","last_active":"2026-02-22T22:39:34.607296+0000","last_peered":"2026-02-22T22:39:34.607296+0000","last_clean":"2026-02-22T22:39:34.607296+0000","last_became_active":"2026-02-22T22:38:15.574752+0000","last_became_peered":"2026-02-22T22:38:15.574752+0000","last_unstale":"2026-02-22T22:39:34.607296+0000","last_undegraded":"2026-02-22T22:39:34.607296+0000","last_fullsized":"2026-02-22T22:39:34.607296+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:05:37.350245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082003000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607286+0000","last_change":"2026-02-22T22:38:17.576762+0000","last_active":"2026-02-22T22:39:34.607286+0000","last_peered":"2026-02-22T22:39:34.607286+0000","last_clean":"2026-02-22T22:39:34.607286+0000","last_became_active":"2026-02-22T22:38:15.574306+0000","last_became_peered":"2026-02-22T22:38:15.574306+0000","last_unstale":"2026-02-22T22:39:34.607286+0000","last_undegraded":"2026-02-22T22:39:34.607286+0000","last_fullsized":"2026-02-22T22:39:34.607286+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:25:28.811738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059383000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607198+0000","last_change":"2026-02-22T22:38:17.576464+0000","last_active":"2026-02-22T22:39:34.607198+0000","last_peered":"2026-02-22T22:39:34.607198+0000","last_clean":"2026-02-22T22:39:34.607198+0000","last_became_active":"2026-02-22T22:38:15.574673+0000","last_became_peered":"2026-02-22T22:38:15.574673+0000","last_unstale":"2026-02-22T22:39:34.607198+0000","last_undegraded":"2026-02-22T22:39:34.607198+0000","last_fullsized":"2026-02-22T22:39:34.607198+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:46.550560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031834000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606534+0000","last_change":"2026-02-22T22:38:17.576467+0000","last_active":"2026-02-22T22:39:34.606534+0000","last_peered":"2026-02-22T22:39:34.606534+0000","last_clean":"2026-02-22T22:39:34.606534+0000","last_became_active":"2026-02-22T22:38:15.574763+0000","last_became_peered":"2026-02-22T22:38:15.574763+0000","last_unstale":"2026-02-22T22:39:34.606534+0000","last_undegraded":"2026-02-22T22:39:34.606534+0000","last_fullsized":"2026-02-22T22:39:34.606534+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:25.430048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031511400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":76,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.557968+0000","last_change":"2026-02-22T22:38:17.575058+0000","last_active":"2026-02-22T22:39:34.557968+0000","last_peered":"2026-02-22T22:39:34.557968+0000","last_clean":"2026-02-22T22:39:34.557968+0000","last_became_active":"2026-02-22T22:38:15.574409+0000","last_became_peered":"2026-02-22T22:38:15.574409+0000","last_unstale":"2026-02-22T22:39:34.557968+0000","last_undegraded":"2026-02-22T22:39:34.557968+0000","last_fullsized":"2026-02-22T22:39:34.557968+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:32:34.479960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053028299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606220+0000","last_change":"2026-02-22T22:38:17.677203+0000","last_active":"2026-02-22T22:39:34.606220+0000","last_peered":"2026-02-22T22:39:34.606220+0000","last_clean":"2026-02-22T22:39:34.606220+0000","last_became_active":"2026-02-22T22:38:15.574389+0000","last_became_peered":"2026-02-22T22:38:15.574389+0000","last_unstale":"2026-02-22T22:39:34.606220+0000","last_undegraded":"2026-02-22T22:39:34.606220+0000","last_fullsized":"2026-02-22T22:39:34.606220+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:09:24.494810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021430700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.605887+0000","last_change":"2026-02-22T22:38:17.677186+0000","last_active":"2026-02-22T22:39:34.605887+0000","last_peered":"2026-02-22T22:39:34.605887+0000","last_clean":"2026-02-22T22:39:34.605887+0000","last_became_active":"2026-02-22T22:38:15.574402+0000","last_became_peered":"2026-02-22T22:38:15.574402+0000","last_unstale":"2026-02-22T22:39:34.605887+0000","last_undegraded":"2026-02-22T22:39:34.605887+0000","last_fullsized":"2026-02-22T22:39:34.605887+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:19.472593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019295799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606727+0000","last_change":"2026-02-22T22:38:17.576831+0000","last_active":"2026-02-22T22:39:34.606727+0000","last_peered":"2026-02-22T22:39:34.606727+0000","last_clean":"2026-02-22T22:39:34.606727+0000","last_became_active":"2026-02-22T22:38:15.574263+0000","last_became_peered":"2026-02-22T22:38:15.574263+0000","last_unstale":"2026-02-22T22:39:34.606727+0000","last_undegraded":"2026-02-22T22:39:34.606727+0000","last_fullsized":"2026-02-22T22:39:34.606727+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:41.940013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068121299999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":165,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.605740+0000","last_change":"2026-02-22T22:38:11.502395+0000","last_active":"2026-02-22T22:39:34.605740+0000","last_peered":"2026-02-22T22:39:34.605740+0000","last_clean":"2026-02-22T22:39:34.605740+0000","last_became_active":"2026-02-22T22:38:11.501894+0000","last_became_peered":"2026-02-22T22:38:11.501894+0000","last_unstale":"2026-02-22T22:39:34.605740+0000","last_undegraded":"2026-02-22T22:39:34.605740+0000","last_fullsized":"2026-02-22T22:39:34.605740+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_clean_scrub_stamp":"2026-02-22T22:38:09.558114+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:19:00.206795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705689,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":29584,"kb_used_data":2624,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94342256,"statfs":{"total":96636764160,"available":96606470144,"internally_reserved":0,"allocated":2686976,"data_stored":2444772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738394,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":30168,"kb_used_data":3208,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94341672,"statfs":{"total":96636764160,"available":96605872128,"internally_reserved":0,"allocated":3284992,"data_stored":3014165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738394,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":29936,"kb_used_data":2976,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94341904,"statfs":{"total":96636764160,"available":96606109696,"internally_reserved":0,"allocated":3047424,"data_stored":2783202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:04.753 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:40:04.753 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:40:04.912 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:40:04.912 INFO:teuthology.orchestra.run.trial178.stderr:dumped all 2026-02-22T22:40:04.921 INFO:teuthology.orchestra.run.trial178.stdout:{"pg_ready":true,"pg_map":{"version":100,"stamp":"2026-02-22T22:40:03.438915+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":89688,"kb_used_data":8808,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283025832,"statfs":{"total":289910292480,"available":289818451968,"internally_reserved":0,"allocated":9019392,"data_stored":8242139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-558846990,"num_objects":-244,"num_object_clones":0,"num_object_copies":-488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-244,"num_whiteouts":0,"num_read":-4546,"num_read_kb":-5342,"num_write":-1011,"num_write_kb":-3545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.000848"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607296+0000","last_change":"2026-02-22T22:38:17.577006+0000","last_active":"2026-02-22T22:39:34.607296+0000","last_peered":"2026-02-22T22:39:34.607296+0000","last_clean":"2026-02-22T22:39:34.607296+0000","last_became_active":"2026-02-22T22:38:15.574752+0000","last_became_peered":"2026-02-22T22:38:15.574752+0000","last_unstale":"2026-02-22T22:39:34.607296+0000","last_undegraded":"2026-02-22T22:39:34.607296+0000","last_fullsized":"2026-02-22T22:39:34.607296+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:05:37.350245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082003000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607286+0000","last_change":"2026-02-22T22:38:17.576762+0000","last_active":"2026-02-22T22:39:34.607286+0000","last_peered":"2026-02-22T22:39:34.607286+0000","last_clean":"2026-02-22T22:39:34.607286+0000","last_became_active":"2026-02-22T22:38:15.574306+0000","last_became_peered":"2026-02-22T22:38:15.574306+0000","last_unstale":"2026-02-22T22:39:34.607286+0000","last_undegraded":"2026-02-22T22:39:34.607286+0000","last_fullsized":"2026-02-22T22:39:34.607286+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:25:28.811738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059383000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.607198+0000","last_change":"2026-02-22T22:38:17.576464+0000","last_active":"2026-02-22T22:39:34.607198+0000","last_peered":"2026-02-22T22:39:34.607198+0000","last_clean":"2026-02-22T22:39:34.607198+0000","last_became_active":"2026-02-22T22:38:15.574673+0000","last_became_peered":"2026-02-22T22:38:15.574673+0000","last_unstale":"2026-02-22T22:39:34.607198+0000","last_undegraded":"2026-02-22T22:39:34.607198+0000","last_fullsized":"2026-02-22T22:39:34.607198+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:46.550560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031834000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":74,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606534+0000","last_change":"2026-02-22T22:38:17.576467+0000","last_active":"2026-02-22T22:39:34.606534+0000","last_peered":"2026-02-22T22:39:34.606534+0000","last_clean":"2026-02-22T22:39:34.606534+0000","last_became_active":"2026-02-22T22:38:15.574763+0000","last_became_peered":"2026-02-22T22:38:15.574763+0000","last_unstale":"2026-02-22T22:39:34.606534+0000","last_undegraded":"2026-02-22T22:39:34.606534+0000","last_fullsized":"2026-02-22T22:39:34.606534+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:25.430048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031511400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":76,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.557968+0000","last_change":"2026-02-22T22:38:17.575058+0000","last_active":"2026-02-22T22:39:34.557968+0000","last_peered":"2026-02-22T22:39:34.557968+0000","last_clean":"2026-02-22T22:39:34.557968+0000","last_became_active":"2026-02-22T22:38:15.574409+0000","last_became_peered":"2026-02-22T22:38:15.574409+0000","last_unstale":"2026-02-22T22:39:34.557968+0000","last_undegraded":"2026-02-22T22:39:34.557968+0000","last_fullsized":"2026-02-22T22:39:34.557968+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:32:34.479960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053028299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606220+0000","last_change":"2026-02-22T22:38:17.677203+0000","last_active":"2026-02-22T22:39:34.606220+0000","last_peered":"2026-02-22T22:39:34.606220+0000","last_clean":"2026-02-22T22:39:34.606220+0000","last_became_active":"2026-02-22T22:38:15.574389+0000","last_became_peered":"2026-02-22T22:38:15.574389+0000","last_unstale":"2026-02-22T22:39:34.606220+0000","last_undegraded":"2026-02-22T22:39:34.606220+0000","last_fullsized":"2026-02-22T22:39:34.606220+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:09:24.494810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021430700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.605887+0000","last_change":"2026-02-22T22:38:17.677186+0000","last_active":"2026-02-22T22:39:34.605887+0000","last_peered":"2026-02-22T22:39:34.605887+0000","last_clean":"2026-02-22T22:39:34.605887+0000","last_became_active":"2026-02-22T22:38:15.574402+0000","last_became_peered":"2026-02-22T22:38:15.574402+0000","last_unstale":"2026-02-22T22:39:34.605887+0000","last_undegraded":"2026-02-22T22:39:34.605887+0000","last_fullsized":"2026-02-22T22:39:34.605887+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:19.472593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019295799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":75,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.606727+0000","last_change":"2026-02-22T22:38:17.576831+0000","last_active":"2026-02-22T22:39:34.606727+0000","last_peered":"2026-02-22T22:39:34.606727+0000","last_clean":"2026-02-22T22:39:34.606727+0000","last_became_active":"2026-02-22T22:38:15.574263+0000","last_became_peered":"2026-02-22T22:38:15.574263+0000","last_unstale":"2026-02-22T22:39:34.606727+0000","last_undegraded":"2026-02-22T22:39:34.606727+0000","last_fullsized":"2026-02-22T22:39:34.606727+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:41.940013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068121299999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":165,"reported_epoch":43,"state":"active+clean","last_fresh":"2026-02-22T22:39:34.605740+0000","last_change":"2026-02-22T22:38:11.502395+0000","last_active":"2026-02-22T22:39:34.605740+0000","last_peered":"2026-02-22T22:39:34.605740+0000","last_clean":"2026-02-22T22:39:34.605740+0000","last_became_active":"2026-02-22T22:38:11.501894+0000","last_became_peered":"2026-02-22T22:38:11.501894+0000","last_unstale":"2026-02-22T22:39:34.605740+0000","last_undegraded":"2026-02-22T22:39:34.605740+0000","last_fullsized":"2026-02-22T22:39:34.605740+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_clean_scrub_stamp":"2026-02-22T22:38:09.558114+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:19:00.206795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705689,"num_pgs":80,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":29584,"kb_used_data":2624,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94342256,"statfs":{"total":96636764160,"available":96606470144,"internally_reserved":0,"allocated":2686976,"data_stored":2444772,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738394,"num_pgs":98,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":30168,"kb_used_data":3208,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94341672,"statfs":{"total":96636764160,"available":96605872128,"internally_reserved":0,"allocated":3284992,"data_stored":3014165,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738394,"num_pgs":96,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":29936,"kb_used_data":2976,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94341904,"statfs":{"total":96636764160,"available":96606109696,"internally_reserved":0,"allocated":3047424,"data_stored":2783202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:04.922 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:40:05.082 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:40:05.083 INFO:teuthology.orchestra.run.trial178.stdout:{"epoch":45,"fsid":"24cee112-44da-4e56-845c-136a54a559f9","created":"2026-02-22T22:37:59.549109+0000","modified":"2026-02-22T22:40:02.574579+0000","last_up_change":"2026-02-22T22:38:09.558114+0000","last_in_change":"2026-02-22T22:38:00.132360+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:38:09.439442+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"11","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:38:13.744265+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"15","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":15,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"f2b02fcf-c25e-422e-bed4-e1b897f6ec53","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":24,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6800","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6801","nonce":184969568}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6802","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6803","nonce":184969568}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6806","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6807","nonce":184969568}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6804","nonce":184969568},{"type":"v1","addr":"10.20.193.178:6805","nonce":184969568}]},"public_addr":"10.20.193.178:6801/184969568","cluster_addr":"10.20.193.178:6803/184969568","heartbeat_back_addr":"10.20.193.178:6807/184969568","heartbeat_front_addr":"10.20.193.178:6805/184969568","state":["exists","up"]},{"osd":1,"uuid":"b2a9d9d6-5251-46f8-8eea-9cb5b8a7d7fb","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":18,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6808","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6809","nonce":3461381058}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6810","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6811","nonce":3461381058}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6814","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6815","nonce":3461381058}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6812","nonce":3461381058},{"type":"v1","addr":"10.20.193.178:6813","nonce":3461381058}]},"public_addr":"10.20.193.178:6809/3461381058","cluster_addr":"10.20.193.178:6811/3461381058","heartbeat_back_addr":"10.20.193.178:6815/3461381058","heartbeat_front_addr":"10.20.193.178:6813/3461381058","state":["exists","up"]},{"osd":2,"uuid":"f28d8f58-67be-4316-bb40-0fad368ef7fe","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":24,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6816","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6817","nonce":371009371}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6818","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6819","nonce":371009371}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6822","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6823","nonce":371009371}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.178:6820","nonce":371009371},{"type":"v1","addr":"10.20.193.178:6821","nonce":371009371}]},"public_addr":"10.20.193.178:6817/371009371","cluster_addr":"10.20.193.178:6819/371009371","heartbeat_back_addr":"10.20.193.178:6823/371009371","heartbeat_front_addr":"10.20.193.178:6821/371009371","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.496134+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.544189+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:38:07.582806+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:40:06.093 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-22T22:40:06.093 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:40:06.178 INFO:teuthology.orchestra.run.trial178.stdout:{ 2026-02-22T22:40:06.179 INFO:teuthology.orchestra.run.trial178.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:40:06.179 INFO:teuthology.orchestra.run.trial178.stdout:} 2026-02-22T22:40:06.184 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-22T22:40:06.343 INFO:teuthology.orchestra.run.trial178.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:40:06.352 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-22T22:40:06.352 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:40:06.432 INFO:teuthology.orchestra.run.trial178.stdout:{ 2026-02-22T22:40:06.432 INFO:teuthology.orchestra.run.trial178.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:40:06.432 INFO:teuthology.orchestra.run.trial178.stdout:} 2026-02-22T22:40:06.437 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-22T22:40:06.597 INFO:teuthology.orchestra.run.trial178.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:40:06.606 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-22T22:40:06.607 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:40:06.688 INFO:teuthology.orchestra.run.trial178.stdout:{ 2026-02-22T22:40:06.688 INFO:teuthology.orchestra.run.trial178.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:40:06.688 INFO:teuthology.orchestra.run.trial178.stdout:} 2026-02-22T22:40:06.694 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-22T22:40:06.853 INFO:teuthology.orchestra.run.trial178.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:40:06.864 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:40:07.020 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:40:07.021 INFO:teuthology.orchestra.run.trial178.stderr:dumped all 2026-02-22T22:40:07.028 INFO:teuthology.orchestra.run.trial178.stdout:{"pg_ready":true,"pg_map":{"version":101,"stamp":"2026-02-22T22:40:05.439460+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":146,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":91516,"kb_used_data":2296,"kb_used_omap":4,"kb_used_meta":89147,"kb_avail":283024004,"statfs":{"total":289910292480,"available":289816580096,"internally_reserved":0,"allocated":2351104,"data_stored":1685077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":91286866},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-558846990,"num_objects":-244,"num_object_clones":0,"num_object_copies":-488,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-244,"num_whiteouts":0,"num_read":-4546,"num_read_kb":-5342,"num_write":-1011,"num_write_kb":-3545,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001065"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:03.650802+0000","last_change":"2026-02-22T22:38:17.577006+0000","last_active":"2026-02-22T22:40:03.650802+0000","last_peered":"2026-02-22T22:40:03.650802+0000","last_clean":"2026-02-22T22:40:03.650802+0000","last_became_active":"2026-02-22T22:38:15.574752+0000","last_became_peered":"2026-02-22T22:38:15.574752+0000","last_unstale":"2026-02-22T22:40:03.650802+0000","last_undegraded":"2026-02-22T22:40:03.650802+0000","last_fullsized":"2026-02-22T22:40:03.650802+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:05:37.350245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082003000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:03.650788+0000","last_change":"2026-02-22T22:38:17.576762+0000","last_active":"2026-02-22T22:40:03.650788+0000","last_peered":"2026-02-22T22:40:03.650788+0000","last_clean":"2026-02-22T22:40:03.650788+0000","last_became_active":"2026-02-22T22:38:15.574306+0000","last_became_peered":"2026-02-22T22:38:15.574306+0000","last_unstale":"2026-02-22T22:40:03.650788+0000","last_undegraded":"2026-02-22T22:40:03.650788+0000","last_fullsized":"2026-02-22T22:40:03.650788+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:25:28.811738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059383000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:03.651728+0000","last_change":"2026-02-22T22:38:17.576464+0000","last_active":"2026-02-22T22:40:03.651728+0000","last_peered":"2026-02-22T22:40:03.651728+0000","last_clean":"2026-02-22T22:40:03.651728+0000","last_became_active":"2026-02-22T22:38:15.574673+0000","last_became_peered":"2026-02-22T22:38:15.574673+0000","last_unstale":"2026-02-22T22:40:03.651728+0000","last_undegraded":"2026-02-22T22:40:03.651728+0000","last_fullsized":"2026-02-22T22:40:03.651728+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:39:46.550560+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031834000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:03.650662+0000","last_change":"2026-02-22T22:38:17.576467+0000","last_active":"2026-02-22T22:40:03.650662+0000","last_peered":"2026-02-22T22:40:03.650662+0000","last_clean":"2026-02-22T22:40:03.650662+0000","last_became_active":"2026-02-22T22:38:15.574763+0000","last_became_peered":"2026-02-22T22:38:15.574763+0000","last_unstale":"2026-02-22T22:40:03.650662+0000","last_undegraded":"2026-02-22T22:40:03.650662+0000","last_fullsized":"2026-02-22T22:40:03.650662+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:33:25.430048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031511400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"15'2","reported_seq":80,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:03.011345+0000","last_change":"2026-02-22T22:38:17.575058+0000","last_active":"2026-02-22T22:40:03.011345+0000","last_peered":"2026-02-22T22:40:03.011345+0000","last_clean":"2026-02-22T22:40:03.011345+0000","last_became_active":"2026-02-22T22:38:15.574409+0000","last_became_peered":"2026-02-22T22:38:15.574409+0000","last_unstale":"2026-02-22T22:40:03.011345+0000","last_undegraded":"2026-02-22T22:40:03.011345+0000","last_fullsized":"2026-02-22T22:40:03.011345+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:32:34.479960+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00053028299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":79,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:02.580669+0000","last_change":"2026-02-22T22:38:17.677203+0000","last_active":"2026-02-22T22:40:02.580669+0000","last_peered":"2026-02-22T22:40:02.580669+0000","last_clean":"2026-02-22T22:40:02.580669+0000","last_became_active":"2026-02-22T22:38:15.574389+0000","last_became_peered":"2026-02-22T22:38:15.574389+0000","last_unstale":"2026-02-22T22:40:02.580669+0000","last_undegraded":"2026-02-22T22:40:02.580669+0000","last_fullsized":"2026-02-22T22:40:02.580669+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:09:24.494810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021430700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":79,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:02.580350+0000","last_change":"2026-02-22T22:38:17.677186+0000","last_active":"2026-02-22T22:40:02.580350+0000","last_peered":"2026-02-22T22:40:02.580350+0000","last_clean":"2026-02-22T22:40:02.580350+0000","last_became_active":"2026-02-22T22:38:15.574402+0000","last_became_peered":"2026-02-22T22:38:15.574402+0000","last_unstale":"2026-02-22T22:40:02.580350+0000","last_undegraded":"2026-02-22T22:40:02.580350+0000","last_fullsized":"2026-02-22T22:40:02.580350+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:19.472593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019295799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"13'1","reported_seq":79,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:03.651022+0000","last_change":"2026-02-22T22:38:17.576831+0000","last_active":"2026-02-22T22:40:03.651022+0000","last_peered":"2026-02-22T22:40:03.651022+0000","last_clean":"2026-02-22T22:40:03.651022+0000","last_became_active":"2026-02-22T22:38:15.574263+0000","last_became_peered":"2026-02-22T22:38:15.574263+0000","last_unstale":"2026-02-22T22:40:03.651022+0000","last_undegraded":"2026-02-22T22:40:03.651022+0000","last_fullsized":"2026-02-22T22:40:03.651022+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:14.565726+0000","last_clean_scrub_stamp":"2026-02-22T22:38:14.565726+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:11:41.940013+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068121299999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":169,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:03.648218+0000","last_change":"2026-02-22T22:38:11.502395+0000","last_active":"2026-02-22T22:40:03.648218+0000","last_peered":"2026-02-22T22:40:03.648218+0000","last_clean":"2026-02-22T22:40:03.648218+0000","last_became_active":"2026-02-22T22:38:11.501894+0000","last_became_peered":"2026-02-22T22:38:11.501894+0000","last_unstale":"2026-02-22T22:40:03.648218+0000","last_undegraded":"2026-02-22T22:40:03.648218+0000","last_fullsized":"2026-02-22T22:40:03.648218+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:38:09.558114+0000","last_clean_scrub_stamp":"2026-02-22T22:38:09.558114+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:19:00.206795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705690,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27344,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344496,"statfs":{"total":96636764160,"available":96608763904,"internally_reserved":0,"allocated":385024,"data_stored":168101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738395,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32088,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339752,"statfs":{"total":96636764160,"available":96603906048,"internally_reserved":0,"allocated":983040,"data_stored":758488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738395,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":758488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:07.033 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T22:38:14.565726+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.034 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T22:38:14.565726+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.034 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T22:38:14.565726+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.034 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T22:38:14.565726+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.034 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T22:38:14.565726+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.034 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T22:38:14.565726+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.034 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T22:38:14.565726+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.035 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T22:38:14.565726+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.035 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T22:38:09.558114+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=38, tm_sec=9, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=5, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:07.035 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:40:27.036 DEBUG:teuthology.orchestra.run.trial178:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:40:27.195 INFO:teuthology.orchestra.run.trial178.stdout: 2026-02-22T22:40:27.195 INFO:teuthology.orchestra.run.trial178.stderr:dumped all 2026-02-22T22:40:27.203 INFO:teuthology.orchestra.run.trial178.stdout:{"pg_ready":true,"pg_map":{"version":111,"stamp":"2026-02-22T22:40:25.441019+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95656,"kb_used_data":2296,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283019864,"statfs":{"total":289910292480,"available":289812340736,"internally_reserved":0,"allocated":2351104,"data_stored":1685077,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000881"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":87,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:11.826595+0000","last_change":"2026-02-22T22:40:11.826614+0000","last_active":"2026-02-22T22:40:11.826595+0000","last_peered":"2026-02-22T22:40:11.826595+0000","last_clean":"2026-02-22T22:40:11.826595+0000","last_became_active":"2026-02-22T22:38:15.574752+0000","last_became_peered":"2026-02-22T22:38:15.574752+0000","last_unstale":"2026-02-22T22:40:11.826595+0000","last_undegraded":"2026-02-22T22:40:11.826595+0000","last_fullsized":"2026-02-22T22:40:11.826595+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:11.826535+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:11.826535+0000","last_clean_scrub_stamp":"2026-02-22T22:40:11.826535+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:44:39.526456+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00082003000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":88,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:07.823697+0000","last_change":"2026-02-22T22:40:07.823697+0000","last_active":"2026-02-22T22:40:07.823697+0000","last_peered":"2026-02-22T22:40:07.823697+0000","last_clean":"2026-02-22T22:40:07.823697+0000","last_became_active":"2026-02-22T22:38:15.574306+0000","last_became_peered":"2026-02-22T22:38:15.574306+0000","last_unstale":"2026-02-22T22:40:07.823697+0000","last_undegraded":"2026-02-22T22:40:07.823697+0000","last_fullsized":"2026-02-22T22:40:07.823697+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:07.823649+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:07.823649+0000","last_clean_scrub_stamp":"2026-02-22T22:40:07.823649+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:50:42.206304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059383000000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":88,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:09.780109+0000","last_change":"2026-02-22T22:40:09.780109+0000","last_active":"2026-02-22T22:40:09.780109+0000","last_peered":"2026-02-22T22:40:09.780109+0000","last_clean":"2026-02-22T22:40:09.780109+0000","last_became_active":"2026-02-22T22:38:15.574673+0000","last_became_peered":"2026-02-22T22:38:15.574673+0000","last_unstale":"2026-02-22T22:40:09.780109+0000","last_undegraded":"2026-02-22T22:40:09.780109+0000","last_fullsized":"2026-02-22T22:40:09.780109+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:09.780026+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:09.780026+0000","last_clean_scrub_stamp":"2026-02-22T22:40:09.780026+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:41:17.028852+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031834000000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":88,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:10.796901+0000","last_change":"2026-02-22T22:40:10.796901+0000","last_active":"2026-02-22T22:40:10.796901+0000","last_peered":"2026-02-22T22:40:10.796901+0000","last_clean":"2026-02-22T22:40:10.796901+0000","last_became_active":"2026-02-22T22:38:15.574763+0000","last_became_peered":"2026-02-22T22:38:15.574763+0000","last_unstale":"2026-02-22T22:40:10.796901+0000","last_undegraded":"2026-02-22T22:40:10.796901+0000","last_fullsized":"2026-02-22T22:40:10.796901+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:10.796832+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:10.796832+0000","last_clean_scrub_stamp":"2026-02-22T22:40:10.796832+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:21:16.539047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00031511400000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"45'3","reported_seq":91,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:07.338457+0000","last_change":"2026-02-22T22:40:07.338457+0000","last_active":"2026-02-22T22:40:07.338457+0000","last_peered":"2026-02-22T22:40:07.338457+0000","last_clean":"2026-02-22T22:40:07.338457+0000","last_became_active":"2026-02-22T22:38:15.574409+0000","last_became_peered":"2026-02-22T22:38:15.574409+0000","last_unstale":"2026-02-22T22:40:07.338457+0000","last_undegraded":"2026-02-22T22:40:07.338457+0000","last_fullsized":"2026-02-22T22:40:07.338457+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"45'3","last_scrub_stamp":"2026-02-22T22:40:07.338381+0000","last_deep_scrub":"45'3","last_deep_scrub_stamp":"2026-02-22T22:40:07.338381+0000","last_clean_scrub_stamp":"2026-02-22T22:40:07.338381+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:11:41.298681+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00053028299999999999,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":89,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:07.359407+0000","last_change":"2026-02-22T22:40:07.359407+0000","last_active":"2026-02-22T22:40:07.359407+0000","last_peered":"2026-02-22T22:40:07.359407+0000","last_clean":"2026-02-22T22:40:07.359407+0000","last_became_active":"2026-02-22T22:38:15.574389+0000","last_became_peered":"2026-02-22T22:38:15.574389+0000","last_unstale":"2026-02-22T22:40:07.359407+0000","last_undegraded":"2026-02-22T22:40:07.359407+0000","last_fullsized":"2026-02-22T22:40:07.359407+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:07.359327+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:07.359327+0000","last_clean_scrub_stamp":"2026-02-22T22:40:07.359327+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:53:17.209550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00021430700000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":89,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:08.321703+0000","last_change":"2026-02-22T22:40:08.321703+0000","last_active":"2026-02-22T22:40:08.321703+0000","last_peered":"2026-02-22T22:40:08.321703+0000","last_clean":"2026-02-22T22:40:08.321703+0000","last_became_active":"2026-02-22T22:38:15.574402+0000","last_became_peered":"2026-02-22T22:38:15.574402+0000","last_unstale":"2026-02-22T22:40:08.321703+0000","last_undegraded":"2026-02-22T22:40:08.321703+0000","last_fullsized":"2026-02-22T22:40:08.321703+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:08.321652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:08.321652+0000","last_clean_scrub_stamp":"2026-02-22T22:40:08.321652+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:35:53.033127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00019295799999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"45'2","reported_seq":90,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:08.798457+0000","last_change":"2026-02-22T22:40:08.798457+0000","last_active":"2026-02-22T22:40:08.798457+0000","last_peered":"2026-02-22T22:40:08.798457+0000","last_clean":"2026-02-22T22:40:08.798457+0000","last_became_active":"2026-02-22T22:38:15.574263+0000","last_became_peered":"2026-02-22T22:38:15.574263+0000","last_unstale":"2026-02-22T22:40:08.798457+0000","last_undegraded":"2026-02-22T22:40:08.798457+0000","last_fullsized":"2026-02-22T22:40:08.798457+0000","mapping_epoch":12,"log_start":"0'0","ondisk_log_start":"0'0","created":12,"last_epoch_clean":13,"parent":"0.0","parent_split_bits":0,"last_scrub":"45'2","last_scrub_stamp":"2026-02-22T22:40:08.798418+0000","last_deep_scrub":"45'2","last_deep_scrub_stamp":"2026-02-22T22:40:08.798418+0000","last_clean_scrub_stamp":"2026-02-22T22:40:08.798418+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:30:03.853181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068121299999999996,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"45'74","reported_seq":181,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:06.822384+0000","last_change":"2026-02-22T22:40:06.822384+0000","last_active":"2026-02-22T22:40:06.822384+0000","last_peered":"2026-02-22T22:40:06.822384+0000","last_clean":"2026-02-22T22:40:06.822384+0000","last_became_active":"2026-02-22T22:38:11.501894+0000","last_became_peered":"2026-02-22T22:38:11.501894+0000","last_unstale":"2026-02-22T22:40:06.822384+0000","last_undegraded":"2026-02-22T22:40:06.822384+0000","last_fullsized":"2026-02-22T22:40:06.822384+0000","mapping_epoch":9,"log_start":"0'0","ondisk_log_start":"0'0","created":9,"last_epoch_clean":10,"parent":"0.0","parent_split_bits":0,"last_scrub":"45'74","last_scrub_stamp":"2026-02-22T22:40:06.822340+0000","last_deep_scrub":"45'74","last_deep_scrub_stamp":"2026-02-22T22:40:06.822340+0000","last_clean_scrub_stamp":"2026-02-22T22:40:06.822340+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:54:14.446314+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705694,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31496,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94340344,"statfs":{"total":96636764160,"available":96604512256,"internally_reserved":0,"allocated":385024,"data_stored":168101,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738399,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32080,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339760,"statfs":{"total":96636764160,"available":96603914240,"internally_reserved":0,"allocated":983040,"data_stored":758488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738399,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32080,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339760,"statfs":{"total":96636764160,"available":96603914240,"internally_reserved":0,"allocated":983040,"data_stored":758488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:27.205 DEBUG:teuthology.orchestra.run.trial178:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-22T22:40:27.373 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:40:27.373 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:40:27.373 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-22T22:40:27.374 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:27.405 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-22T22:40:27.405 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-22T22:40:27.405 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:27.471 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-22T22:40:27.471 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-22T22:40:27.471 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:27.542 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-22T22:40:27.543 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:40:27.543 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-22T22:40:27.543 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:27.573 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-22T22:40:27.573 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:40:27.573 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-22T22:40:27.573 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:27.593 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-22T22:40:27.593 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:40:27.593 DEBUG:teuthology.orchestra.run.trial178:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:40:27.662 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:40:27.662 DEBUG:teuthology.orchestra.run.trial178:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-22T22:40:27.733 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:40:27.733 DEBUG:teuthology.orchestra.run.trial178:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-22T22:40:27.800 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:40:27.800 DEBUG:teuthology.orchestra.run.trial178:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-22T22:40:27.869 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:40:27.870 DEBUG:teuthology.misc:Transferring archived files from trial178:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259/data/cluster2.mon.a.tgz 2026-02-22T22:40:27.871 DEBUG:teuthology.orchestra.run.trial178:> mktemp 2026-02-22T22:40:27.925 INFO:teuthology.orchestra.run.trial178.stdout:/tmp/tmp.HxdOzybWzl 2026-02-22T22:40:27.926 DEBUG:teuthology.orchestra.run.trial178:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.HxdOzybWzl 2026-02-22T22:40:28.016 DEBUG:teuthology.orchestra.run.trial178:> sudo chmod 0666 /tmp/tmp.HxdOzybWzl 2026-02-22T22:40:28.096 DEBUG:teuthology.orchestra.remote:trial178:/tmp/tmp.HxdOzybWzl is 211KB 2026-02-22T22:40:28.188 DEBUG:teuthology.orchestra.run.trial178:> rm -fr /tmp/tmp.HxdOzybWzl 2026-02-22T22:40:28.205 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:40:28.205 DEBUG:teuthology.orchestra.run.trial078:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:40:28.208 DEBUG:teuthology.orchestra.run.trial178:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:40:28.270 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:40:28.280 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:40:28.281 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:40:28.438 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:40:28.439 INFO:teuthology.orchestra.run.trial078.stderr:dumped all 2026-02-22T22:40:28.446 INFO:teuthology.orchestra.run.trial078.stdout:{"pg_ready":true,"pg_map":{"version":133,"stamp":"2026-02-22T22:40:26.995109+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":91508,"kb_used_data":2296,"kb_used_omap":4,"kb_used_meta":89147,"kb_avail":283024012,"statfs":{"total":289910292480,"available":289816588288,"internally_reserved":0,"allocated":2351104,"data_stored":1666732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":91286868},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000879"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686277+0000","last_change":"2026-02-22T22:37:36.547270+0000","last_active":"2026-02-22T22:40:01.686277+0000","last_peered":"2026-02-22T22:40:01.686277+0000","last_clean":"2026-02-22T22:40:01.686277+0000","last_became_active":"2026-02-22T22:37:34.543281+0000","last_became_peered":"2026-02-22T22:37:34.543281+0000","last_unstale":"2026-02-22T22:40:01.686277+0000","last_undegraded":"2026-02-22T22:40:01.686277+0000","last_fullsized":"2026-02-22T22:40:01.686277+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:45:13.292837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077929400000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686944+0000","last_change":"2026-02-22T22:37:36.547157+0000","last_active":"2026-02-22T22:40:01.686944+0000","last_peered":"2026-02-22T22:40:01.686944+0000","last_clean":"2026-02-22T22:40:01.686944+0000","last_became_active":"2026-02-22T22:37:34.542879+0000","last_became_peered":"2026-02-22T22:37:34.542879+0000","last_unstale":"2026-02-22T22:40:01.686944+0000","last_undegraded":"2026-02-22T22:40:01.686944+0000","last_fullsized":"2026-02-22T22:40:01.686944+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:42:20.635275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070406400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686943+0000","last_change":"2026-02-22T22:37:36.546965+0000","last_active":"2026-02-22T22:40:01.686943+0000","last_peered":"2026-02-22T22:40:01.686943+0000","last_clean":"2026-02-22T22:40:01.686943+0000","last_became_active":"2026-02-22T22:37:34.543262+0000","last_became_peered":"2026-02-22T22:37:34.543262+0000","last_unstale":"2026-02-22T22:40:01.686943+0000","last_undegraded":"2026-02-22T22:40:01.686943+0000","last_fullsized":"2026-02-22T22:40:01.686943+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:37:36.614016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059131299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.685686+0000","last_change":"2026-02-22T22:37:36.547113+0000","last_active":"2026-02-22T22:40:01.685686+0000","last_peered":"2026-02-22T22:40:01.685686+0000","last_clean":"2026-02-22T22:40:01.685686+0000","last_became_active":"2026-02-22T22:37:34.543245+0000","last_became_peered":"2026-02-22T22:37:34.543245+0000","last_unstale":"2026-02-22T22:40:01.685686+0000","last_undegraded":"2026-02-22T22:40:01.685686+0000","last_fullsized":"2026-02-22T22:40:01.685686+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:06:29.211864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068412599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:02.778653+0000","last_change":"2026-02-22T22:37:36.545375+0000","last_active":"2026-02-22T22:40:02.778653+0000","last_peered":"2026-02-22T22:40:02.778653+0000","last_clean":"2026-02-22T22:40:02.778653+0000","last_became_active":"2026-02-22T22:37:34.543078+0000","last_became_peered":"2026-02-22T22:37:34.543078+0000","last_unstale":"2026-02-22T22:40:02.778653+0000","last_undegraded":"2026-02-22T22:40:02.778653+0000","last_fullsized":"2026-02-22T22:40:02.778653+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:44:07.853738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066853699999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:04.359736+0000","last_change":"2026-02-22T22:37:36.542745+0000","last_active":"2026-02-22T22:40:04.359736+0000","last_peered":"2026-02-22T22:40:04.359736+0000","last_clean":"2026-02-22T22:40:04.359736+0000","last_became_active":"2026-02-22T22:37:34.542908+0000","last_became_peered":"2026-02-22T22:37:34.542908+0000","last_unstale":"2026-02-22T22:40:04.359736+0000","last_undegraded":"2026-02-22T22:40:04.359736+0000","last_fullsized":"2026-02-22T22:40:04.359736+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:47:13.400204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043051299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:04.361545+0000","last_change":"2026-02-22T22:37:36.542742+0000","last_active":"2026-02-22T22:40:04.361545+0000","last_peered":"2026-02-22T22:40:04.361545+0000","last_clean":"2026-02-22T22:40:04.361545+0000","last_became_active":"2026-02-22T22:37:34.542961+0000","last_became_peered":"2026-02-22T22:37:34.542961+0000","last_unstale":"2026-02-22T22:40:04.361545+0000","last_undegraded":"2026-02-22T22:40:04.361545+0000","last_fullsized":"2026-02-22T22:40:04.361545+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:14.729623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000344994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":79,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.685624+0000","last_change":"2026-02-22T22:37:36.546970+0000","last_active":"2026-02-22T22:40:01.685624+0000","last_peered":"2026-02-22T22:40:01.685624+0000","last_clean":"2026-02-22T22:40:01.685624+0000","last_became_active":"2026-02-22T22:37:34.542584+0000","last_became_peered":"2026-02-22T22:37:34.542584+0000","last_unstale":"2026-02-22T22:40:01.685624+0000","last_undegraded":"2026-02-22T22:40:01.685624+0000","last_fullsized":"2026-02-22T22:40:01.685624+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:43:40.952490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060424800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":169,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686513+0000","last_change":"2026-02-22T22:37:30.530613+0000","last_active":"2026-02-22T22:40:01.686513+0000","last_peered":"2026-02-22T22:40:01.686513+0000","last_clean":"2026-02-22T22:40:01.686513+0000","last_became_active":"2026-02-22T22:37:30.530398+0000","last_became_peered":"2026-02-22T22:37:30.530398+0000","last_unstale":"2026-02-22T22:40:01.686513+0000","last_undegraded":"2026-02-22T22:40:01.686513+0000","last_fullsized":"2026-02-22T22:40:01.686513+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_clean_scrub_stamp":"2026-02-22T22:37:29.526086+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:41:58.671728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":0,"up_from":9,"seq":38654705703,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705702,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":385024,"data_stored":161986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27589064},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738405,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:28.448 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:40:28.607 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:40:28.607 INFO:teuthology.orchestra.run.trial078.stderr:dumped all 2026-02-22T22:40:28.616 INFO:teuthology.orchestra.run.trial078.stdout:{"pg_ready":true,"pg_map":{"version":133,"stamp":"2026-02-22T22:40:26.995109+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":91508,"kb_used_data":2296,"kb_used_omap":4,"kb_used_meta":89147,"kb_avail":283024012,"statfs":{"total":289910292480,"available":289816588288,"internally_reserved":0,"allocated":2351104,"data_stored":1666732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":91286868},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000879"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686277+0000","last_change":"2026-02-22T22:37:36.547270+0000","last_active":"2026-02-22T22:40:01.686277+0000","last_peered":"2026-02-22T22:40:01.686277+0000","last_clean":"2026-02-22T22:40:01.686277+0000","last_became_active":"2026-02-22T22:37:34.543281+0000","last_became_peered":"2026-02-22T22:37:34.543281+0000","last_unstale":"2026-02-22T22:40:01.686277+0000","last_undegraded":"2026-02-22T22:40:01.686277+0000","last_fullsized":"2026-02-22T22:40:01.686277+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:45:13.292837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077929400000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686944+0000","last_change":"2026-02-22T22:37:36.547157+0000","last_active":"2026-02-22T22:40:01.686944+0000","last_peered":"2026-02-22T22:40:01.686944+0000","last_clean":"2026-02-22T22:40:01.686944+0000","last_became_active":"2026-02-22T22:37:34.542879+0000","last_became_peered":"2026-02-22T22:37:34.542879+0000","last_unstale":"2026-02-22T22:40:01.686944+0000","last_undegraded":"2026-02-22T22:40:01.686944+0000","last_fullsized":"2026-02-22T22:40:01.686944+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:42:20.635275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070406400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686943+0000","last_change":"2026-02-22T22:37:36.546965+0000","last_active":"2026-02-22T22:40:01.686943+0000","last_peered":"2026-02-22T22:40:01.686943+0000","last_clean":"2026-02-22T22:40:01.686943+0000","last_became_active":"2026-02-22T22:37:34.543262+0000","last_became_peered":"2026-02-22T22:37:34.543262+0000","last_unstale":"2026-02-22T22:40:01.686943+0000","last_undegraded":"2026-02-22T22:40:01.686943+0000","last_fullsized":"2026-02-22T22:40:01.686943+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:37:36.614016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059131299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.685686+0000","last_change":"2026-02-22T22:37:36.547113+0000","last_active":"2026-02-22T22:40:01.685686+0000","last_peered":"2026-02-22T22:40:01.685686+0000","last_clean":"2026-02-22T22:40:01.685686+0000","last_became_active":"2026-02-22T22:37:34.543245+0000","last_became_peered":"2026-02-22T22:37:34.543245+0000","last_unstale":"2026-02-22T22:40:01.685686+0000","last_undegraded":"2026-02-22T22:40:01.685686+0000","last_fullsized":"2026-02-22T22:40:01.685686+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:06:29.211864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068412599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:02.778653+0000","last_change":"2026-02-22T22:37:36.545375+0000","last_active":"2026-02-22T22:40:02.778653+0000","last_peered":"2026-02-22T22:40:02.778653+0000","last_clean":"2026-02-22T22:40:02.778653+0000","last_became_active":"2026-02-22T22:37:34.543078+0000","last_became_peered":"2026-02-22T22:37:34.543078+0000","last_unstale":"2026-02-22T22:40:02.778653+0000","last_undegraded":"2026-02-22T22:40:02.778653+0000","last_fullsized":"2026-02-22T22:40:02.778653+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:44:07.853738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066853699999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:04.359736+0000","last_change":"2026-02-22T22:37:36.542745+0000","last_active":"2026-02-22T22:40:04.359736+0000","last_peered":"2026-02-22T22:40:04.359736+0000","last_clean":"2026-02-22T22:40:04.359736+0000","last_became_active":"2026-02-22T22:37:34.542908+0000","last_became_peered":"2026-02-22T22:37:34.542908+0000","last_unstale":"2026-02-22T22:40:04.359736+0000","last_undegraded":"2026-02-22T22:40:04.359736+0000","last_fullsized":"2026-02-22T22:40:04.359736+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:47:13.400204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043051299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:04.361545+0000","last_change":"2026-02-22T22:37:36.542742+0000","last_active":"2026-02-22T22:40:04.361545+0000","last_peered":"2026-02-22T22:40:04.361545+0000","last_clean":"2026-02-22T22:40:04.361545+0000","last_became_active":"2026-02-22T22:37:34.542961+0000","last_became_peered":"2026-02-22T22:37:34.542961+0000","last_unstale":"2026-02-22T22:40:04.361545+0000","last_undegraded":"2026-02-22T22:40:04.361545+0000","last_fullsized":"2026-02-22T22:40:04.361545+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:14.729623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000344994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":79,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.685624+0000","last_change":"2026-02-22T22:37:36.546970+0000","last_active":"2026-02-22T22:40:01.685624+0000","last_peered":"2026-02-22T22:40:01.685624+0000","last_clean":"2026-02-22T22:40:01.685624+0000","last_became_active":"2026-02-22T22:37:34.542584+0000","last_became_peered":"2026-02-22T22:37:34.542584+0000","last_unstale":"2026-02-22T22:40:01.685624+0000","last_undegraded":"2026-02-22T22:40:01.685624+0000","last_fullsized":"2026-02-22T22:40:01.685624+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:43:40.952490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060424800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":169,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686513+0000","last_change":"2026-02-22T22:37:30.530613+0000","last_active":"2026-02-22T22:40:01.686513+0000","last_peered":"2026-02-22T22:40:01.686513+0000","last_clean":"2026-02-22T22:40:01.686513+0000","last_became_active":"2026-02-22T22:37:30.530398+0000","last_became_peered":"2026-02-22T22:37:30.530398+0000","last_unstale":"2026-02-22T22:40:01.686513+0000","last_undegraded":"2026-02-22T22:40:01.686513+0000","last_fullsized":"2026-02-22T22:40:01.686513+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_clean_scrub_stamp":"2026-02-22T22:37:29.526086+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:41:58.671728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":0,"up_from":9,"seq":38654705703,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705702,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":385024,"data_stored":161986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27589064},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738405,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:28.617 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:40:28.618 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:40:28.775 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:40:28.775 INFO:teuthology.orchestra.run.trial078.stderr:dumped all 2026-02-22T22:40:28.784 INFO:teuthology.orchestra.run.trial078.stdout:{"pg_ready":true,"pg_map":{"version":133,"stamp":"2026-02-22T22:40:26.995109+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":91508,"kb_used_data":2296,"kb_used_omap":4,"kb_used_meta":89147,"kb_avail":283024012,"statfs":{"total":289910292480,"available":289816588288,"internally_reserved":0,"allocated":2351104,"data_stored":1666732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":91286868},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000879"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686277+0000","last_change":"2026-02-22T22:37:36.547270+0000","last_active":"2026-02-22T22:40:01.686277+0000","last_peered":"2026-02-22T22:40:01.686277+0000","last_clean":"2026-02-22T22:40:01.686277+0000","last_became_active":"2026-02-22T22:37:34.543281+0000","last_became_peered":"2026-02-22T22:37:34.543281+0000","last_unstale":"2026-02-22T22:40:01.686277+0000","last_undegraded":"2026-02-22T22:40:01.686277+0000","last_fullsized":"2026-02-22T22:40:01.686277+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:45:13.292837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077929400000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686944+0000","last_change":"2026-02-22T22:37:36.547157+0000","last_active":"2026-02-22T22:40:01.686944+0000","last_peered":"2026-02-22T22:40:01.686944+0000","last_clean":"2026-02-22T22:40:01.686944+0000","last_became_active":"2026-02-22T22:37:34.542879+0000","last_became_peered":"2026-02-22T22:37:34.542879+0000","last_unstale":"2026-02-22T22:40:01.686944+0000","last_undegraded":"2026-02-22T22:40:01.686944+0000","last_fullsized":"2026-02-22T22:40:01.686944+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:42:20.635275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070406400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686943+0000","last_change":"2026-02-22T22:37:36.546965+0000","last_active":"2026-02-22T22:40:01.686943+0000","last_peered":"2026-02-22T22:40:01.686943+0000","last_clean":"2026-02-22T22:40:01.686943+0000","last_became_active":"2026-02-22T22:37:34.543262+0000","last_became_peered":"2026-02-22T22:37:34.543262+0000","last_unstale":"2026-02-22T22:40:01.686943+0000","last_undegraded":"2026-02-22T22:40:01.686943+0000","last_fullsized":"2026-02-22T22:40:01.686943+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:37:36.614016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059131299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.685686+0000","last_change":"2026-02-22T22:37:36.547113+0000","last_active":"2026-02-22T22:40:01.685686+0000","last_peered":"2026-02-22T22:40:01.685686+0000","last_clean":"2026-02-22T22:40:01.685686+0000","last_became_active":"2026-02-22T22:37:34.543245+0000","last_became_peered":"2026-02-22T22:37:34.543245+0000","last_unstale":"2026-02-22T22:40:01.685686+0000","last_undegraded":"2026-02-22T22:40:01.685686+0000","last_fullsized":"2026-02-22T22:40:01.685686+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:06:29.211864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068412599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:02.778653+0000","last_change":"2026-02-22T22:37:36.545375+0000","last_active":"2026-02-22T22:40:02.778653+0000","last_peered":"2026-02-22T22:40:02.778653+0000","last_clean":"2026-02-22T22:40:02.778653+0000","last_became_active":"2026-02-22T22:37:34.543078+0000","last_became_peered":"2026-02-22T22:37:34.543078+0000","last_unstale":"2026-02-22T22:40:02.778653+0000","last_undegraded":"2026-02-22T22:40:02.778653+0000","last_fullsized":"2026-02-22T22:40:02.778653+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:44:07.853738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066853699999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:04.359736+0000","last_change":"2026-02-22T22:37:36.542745+0000","last_active":"2026-02-22T22:40:04.359736+0000","last_peered":"2026-02-22T22:40:04.359736+0000","last_clean":"2026-02-22T22:40:04.359736+0000","last_became_active":"2026-02-22T22:37:34.542908+0000","last_became_peered":"2026-02-22T22:37:34.542908+0000","last_unstale":"2026-02-22T22:40:04.359736+0000","last_undegraded":"2026-02-22T22:40:04.359736+0000","last_fullsized":"2026-02-22T22:40:04.359736+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:47:13.400204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043051299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:04.361545+0000","last_change":"2026-02-22T22:37:36.542742+0000","last_active":"2026-02-22T22:40:04.361545+0000","last_peered":"2026-02-22T22:40:04.361545+0000","last_clean":"2026-02-22T22:40:04.361545+0000","last_became_active":"2026-02-22T22:37:34.542961+0000","last_became_peered":"2026-02-22T22:37:34.542961+0000","last_unstale":"2026-02-22T22:40:04.361545+0000","last_undegraded":"2026-02-22T22:40:04.361545+0000","last_fullsized":"2026-02-22T22:40:04.361545+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:14.729623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000344994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":79,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.685624+0000","last_change":"2026-02-22T22:37:36.546970+0000","last_active":"2026-02-22T22:40:01.685624+0000","last_peered":"2026-02-22T22:40:01.685624+0000","last_clean":"2026-02-22T22:40:01.685624+0000","last_became_active":"2026-02-22T22:37:34.542584+0000","last_became_peered":"2026-02-22T22:37:34.542584+0000","last_unstale":"2026-02-22T22:40:01.685624+0000","last_undegraded":"2026-02-22T22:40:01.685624+0000","last_fullsized":"2026-02-22T22:40:01.685624+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:43:40.952490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060424800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":169,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686513+0000","last_change":"2026-02-22T22:37:30.530613+0000","last_active":"2026-02-22T22:40:01.686513+0000","last_peered":"2026-02-22T22:40:01.686513+0000","last_clean":"2026-02-22T22:40:01.686513+0000","last_became_active":"2026-02-22T22:37:30.530398+0000","last_became_peered":"2026-02-22T22:37:30.530398+0000","last_unstale":"2026-02-22T22:40:01.686513+0000","last_undegraded":"2026-02-22T22:40:01.686513+0000","last_fullsized":"2026-02-22T22:40:01.686513+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_clean_scrub_stamp":"2026-02-22T22:37:29.526086+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:41:58.671728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":0,"up_from":9,"seq":38654705703,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705702,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":385024,"data_stored":161986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27589064},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738405,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:28.785 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:40:28.945 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:40:28.946 INFO:teuthology.orchestra.run.trial078.stdout:{"epoch":45,"fsid":"892254a3-cc2a-4241-8fd5-9cfdb608522a","created":"2026-02-22T22:37:18.663727+0000","modified":"2026-02-22T22:40:01.681446+0000","last_up_change":"2026-02-22T22:37:28.524307+0000","last_in_change":"2026-02-22T22:37:19.240135+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:37:28.991462+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:37:32.851019+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"b684e760-514f-4f15-bf12-40cad6d17463","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6810","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6811","nonce":3604302546}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6812","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6813","nonce":3604302546}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6816","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6817","nonce":3604302546}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6814","nonce":3604302546},{"type":"v1","addr":"10.20.193.78:6815","nonce":3604302546}]},"public_addr":"10.20.193.78:6811/3604302546","cluster_addr":"10.20.193.78:6813/3604302546","heartbeat_back_addr":"10.20.193.78:6817/3604302546","heartbeat_front_addr":"10.20.193.78:6815/3604302546","state":["exists","up"]},{"osd":1,"uuid":"7f3bfea4-7d3e-4178-a8d2-e6d4cf6fc32c","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":19,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6800","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6801","nonce":1380231211}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6802","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6803","nonce":1380231211}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6806","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6807","nonce":1380231211}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6804","nonce":1380231211},{"type":"v1","addr":"10.20.193.78:6805","nonce":1380231211}]},"public_addr":"10.20.193.78:6801/1380231211","cluster_addr":"10.20.193.78:6803/1380231211","heartbeat_back_addr":"10.20.193.78:6807/1380231211","heartbeat_front_addr":"10.20.193.78:6805/1380231211","state":["exists","up"]},{"osd":2,"uuid":"7b7e764e-186f-44d7-9826-54a235d28187","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":37,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6818","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6819","nonce":2777796395}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6820","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6821","nonce":2777796395}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6824","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6825","nonce":2777796395}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.78:6822","nonce":2777796395},{"type":"v1","addr":"10.20.193.78:6823","nonce":2777796395}]},"public_addr":"10.20.193.78:6819/2777796395","cluster_addr":"10.20.193.78:6821/2777796395","heartbeat_back_addr":"10.20.193.78:6825/2777796395","heartbeat_front_addr":"10.20.193.78:6823/2777796395","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.613807+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.475169+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:37:26.712892+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:40:29.956 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-22T22:40:29.956 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:40:30.038 INFO:teuthology.orchestra.run.trial078.stdout:{ 2026-02-22T22:40:30.038 INFO:teuthology.orchestra.run.trial078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:40:30.038 INFO:teuthology.orchestra.run.trial078.stdout:} 2026-02-22T22:40:30.044 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-22T22:40:30.203 INFO:teuthology.orchestra.run.trial078.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:40:30.212 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-22T22:40:30.212 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:40:30.295 INFO:teuthology.orchestra.run.trial078.stdout:{ 2026-02-22T22:40:30.295 INFO:teuthology.orchestra.run.trial078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:40:30.295 INFO:teuthology.orchestra.run.trial078.stdout:} 2026-02-22T22:40:30.301 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-22T22:40:30.459 INFO:teuthology.orchestra.run.trial078.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:40:30.469 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-22T22:40:30.469 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:40:30.551 INFO:teuthology.orchestra.run.trial078.stdout:{ 2026-02-22T22:40:30.551 INFO:teuthology.orchestra.run.trial078.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:40:30.551 INFO:teuthology.orchestra.run.trial078.stdout:} 2026-02-22T22:40:30.557 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-22T22:40:30.713 INFO:teuthology.orchestra.run.trial078.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:40:30.722 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:40:30.876 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:40:30.877 INFO:teuthology.orchestra.run.trial078.stderr:dumped all 2026-02-22T22:40:30.884 INFO:teuthology.orchestra.run.trial078.stdout:{"pg_ready":true,"pg_map":{"version":134,"stamp":"2026-02-22T22:40:28.995253+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":91508,"kb_used_data":2296,"kb_used_omap":4,"kb_used_meta":89147,"kb_avail":283024012,"statfs":{"total":289910292480,"available":289816588288,"internally_reserved":0,"allocated":2351104,"data_stored":1666732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":91286868},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000840"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686277+0000","last_change":"2026-02-22T22:37:36.547270+0000","last_active":"2026-02-22T22:40:01.686277+0000","last_peered":"2026-02-22T22:40:01.686277+0000","last_clean":"2026-02-22T22:40:01.686277+0000","last_became_active":"2026-02-22T22:37:34.543281+0000","last_became_peered":"2026-02-22T22:37:34.543281+0000","last_unstale":"2026-02-22T22:40:01.686277+0000","last_undegraded":"2026-02-22T22:40:01.686277+0000","last_fullsized":"2026-02-22T22:40:01.686277+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:45:13.292837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077929400000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686944+0000","last_change":"2026-02-22T22:37:36.547157+0000","last_active":"2026-02-22T22:40:01.686944+0000","last_peered":"2026-02-22T22:40:01.686944+0000","last_clean":"2026-02-22T22:40:01.686944+0000","last_became_active":"2026-02-22T22:37:34.542879+0000","last_became_peered":"2026-02-22T22:37:34.542879+0000","last_unstale":"2026-02-22T22:40:01.686944+0000","last_undegraded":"2026-02-22T22:40:01.686944+0000","last_fullsized":"2026-02-22T22:40:01.686944+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:42:20.635275+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070406400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686943+0000","last_change":"2026-02-22T22:37:36.546965+0000","last_active":"2026-02-22T22:40:01.686943+0000","last_peered":"2026-02-22T22:40:01.686943+0000","last_clean":"2026-02-22T22:40:01.686943+0000","last_became_active":"2026-02-22T22:37:34.543262+0000","last_became_peered":"2026-02-22T22:37:34.543262+0000","last_unstale":"2026-02-22T22:40:01.686943+0000","last_undegraded":"2026-02-22T22:40:01.686943+0000","last_fullsized":"2026-02-22T22:40:01.686943+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:37:36.614016+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059131299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.685686+0000","last_change":"2026-02-22T22:37:36.547113+0000","last_active":"2026-02-22T22:40:01.685686+0000","last_peered":"2026-02-22T22:40:01.685686+0000","last_clean":"2026-02-22T22:40:01.685686+0000","last_became_active":"2026-02-22T22:37:34.543245+0000","last_became_peered":"2026-02-22T22:37:34.543245+0000","last_unstale":"2026-02-22T22:40:01.685686+0000","last_undegraded":"2026-02-22T22:40:01.685686+0000","last_fullsized":"2026-02-22T22:40:01.685686+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:06:29.211864+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068412599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:02.778653+0000","last_change":"2026-02-22T22:37:36.545375+0000","last_active":"2026-02-22T22:40:02.778653+0000","last_peered":"2026-02-22T22:40:02.778653+0000","last_clean":"2026-02-22T22:40:02.778653+0000","last_became_active":"2026-02-22T22:37:34.543078+0000","last_became_peered":"2026-02-22T22:37:34.543078+0000","last_unstale":"2026-02-22T22:40:02.778653+0000","last_undegraded":"2026-02-22T22:40:02.778653+0000","last_fullsized":"2026-02-22T22:40:02.778653+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:44:07.853738+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00066853699999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:04.359736+0000","last_change":"2026-02-22T22:37:36.542745+0000","last_active":"2026-02-22T22:40:04.359736+0000","last_peered":"2026-02-22T22:40:04.359736+0000","last_clean":"2026-02-22T22:40:04.359736+0000","last_became_active":"2026-02-22T22:37:34.542908+0000","last_became_peered":"2026-02-22T22:37:34.542908+0000","last_unstale":"2026-02-22T22:40:04.359736+0000","last_undegraded":"2026-02-22T22:40:04.359736+0000","last_fullsized":"2026-02-22T22:40:04.359736+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T05:47:13.400204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043051299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":78,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:04.361545+0000","last_change":"2026-02-22T22:37:36.542742+0000","last_active":"2026-02-22T22:40:04.361545+0000","last_peered":"2026-02-22T22:40:04.361545+0000","last_clean":"2026-02-22T22:40:04.361545+0000","last_became_active":"2026-02-22T22:37:34.542961+0000","last_became_peered":"2026-02-22T22:37:34.542961+0000","last_unstale":"2026-02-22T22:40:04.361545+0000","last_undegraded":"2026-02-22T22:40:04.361545+0000","last_fullsized":"2026-02-22T22:40:04.361545+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:47:14.729623+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000344994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":79,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.685624+0000","last_change":"2026-02-22T22:37:36.546970+0000","last_active":"2026-02-22T22:40:01.685624+0000","last_peered":"2026-02-22T22:40:01.685624+0000","last_clean":"2026-02-22T22:40:01.685624+0000","last_became_active":"2026-02-22T22:37:34.542584+0000","last_became_peered":"2026-02-22T22:37:34.542584+0000","last_unstale":"2026-02-22T22:40:01.685624+0000","last_undegraded":"2026-02-22T22:40:01.685624+0000","last_fullsized":"2026-02-22T22:40:01.685624+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:33.532641+0000","last_clean_scrub_stamp":"2026-02-22T22:37:33.532641+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:43:40.952490+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060424800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":169,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:01.686513+0000","last_change":"2026-02-22T22:37:30.530613+0000","last_active":"2026-02-22T22:40:01.686513+0000","last_peered":"2026-02-22T22:40:01.686513+0000","last_clean":"2026-02-22T22:40:01.686513+0000","last_became_active":"2026-02-22T22:37:30.530398+0000","last_became_peered":"2026-02-22T22:37:30.530398+0000","last_unstale":"2026-02-22T22:40:01.686513+0000","last_undegraded":"2026-02-22T22:40:01.686513+0000","last_fullsized":"2026-02-22T22:40:01.686513+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:37:29.526086+0000","last_clean_scrub_stamp":"2026-02-22T22:37:29.526086+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:41:58.671728+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":0,"up_from":9,"seq":38654705703,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705703,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27340,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344500,"statfs":{"total":96636764160,"available":96608768000,"internally_reserved":0,"allocated":385024,"data_stored":161986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27589064},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738406,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32084,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339756,"statfs":{"total":96636764160,"available":96603910144,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:30.886 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T22:37:33.532641+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.886 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T22:37:33.532641+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.886 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T22:37:33.532641+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.886 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T22:37:33.532641+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.887 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T22:37:33.532641+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.887 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T22:37:33.532641+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.887 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T22:37:33.532641+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.887 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T22:37:33.532641+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=33, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.887 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T22:37:29.526086+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=37, tm_sec=29, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=40, tm_sec=28, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:40:30.887 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:40:50.888 DEBUG:teuthology.orchestra.run.trial078:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:40:51.050 INFO:teuthology.orchestra.run.trial078.stdout: 2026-02-22T22:40:51.051 INFO:teuthology.orchestra.run.trial078.stderr:dumped all 2026-02-22T22:40:51.059 INFO:teuthology.orchestra.run.trial078.stdout:{"pg_ready":true,"pg_map":{"version":145,"stamp":"2026-02-22T22:40:50.997077+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":91496,"kb_used_data":2296,"kb_used_omap":4,"kb_used_meta":89147,"kb_avail":283024024,"statfs":{"total":289910292480,"available":289816600576,"internally_reserved":0,"allocated":2351104,"data_stored":1666732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4780,"internal_metadata":91286868},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000935"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":87,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:35.826096+0000","last_change":"2026-02-22T22:40:35.826118+0000","last_active":"2026-02-22T22:40:35.826096+0000","last_peered":"2026-02-22T22:40:35.826096+0000","last_clean":"2026-02-22T22:40:35.826096+0000","last_became_active":"2026-02-22T22:37:34.543281+0000","last_became_peered":"2026-02-22T22:37:34.543281+0000","last_unstale":"2026-02-22T22:40:35.826096+0000","last_undegraded":"2026-02-22T22:40:35.826096+0000","last_fullsized":"2026-02-22T22:40:35.826096+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:35.826031+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:35.826031+0000","last_clean_scrub_stamp":"2026-02-22T22:40:35.826031+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:21:41.702924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00077929400000000004,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":87,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:32.842920+0000","last_change":"2026-02-22T22:40:32.842969+0000","last_active":"2026-02-22T22:40:32.842920+0000","last_peered":"2026-02-22T22:40:32.842920+0000","last_clean":"2026-02-22T22:40:32.842920+0000","last_became_active":"2026-02-22T22:37:34.542879+0000","last_became_peered":"2026-02-22T22:37:34.542879+0000","last_unstale":"2026-02-22T22:40:32.842920+0000","last_undegraded":"2026-02-22T22:40:32.842920+0000","last_fullsized":"2026-02-22T22:40:32.842920+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:32.842871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:32.842871+0000","last_clean_scrub_stamp":"2026-02-22T22:40:32.842871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:25:04.189876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00070406400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":87,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:33.832055+0000","last_change":"2026-02-22T22:40:33.832069+0000","last_active":"2026-02-22T22:40:33.832055+0000","last_peered":"2026-02-22T22:40:33.832055+0000","last_clean":"2026-02-22T22:40:33.832055+0000","last_became_active":"2026-02-22T22:37:34.543262+0000","last_became_peered":"2026-02-22T22:37:34.543262+0000","last_unstale":"2026-02-22T22:40:33.832055+0000","last_undegraded":"2026-02-22T22:40:33.832055+0000","last_fullsized":"2026-02-22T22:40:33.832055+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:33.831976+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:33.831976+0000","last_clean_scrub_stamp":"2026-02-22T22:40:33.831976+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:44:31.544119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00059131299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":87,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:34.832940+0000","last_change":"2026-02-22T22:40:34.832955+0000","last_active":"2026-02-22T22:40:34.832940+0000","last_peered":"2026-02-22T22:40:34.832940+0000","last_clean":"2026-02-22T22:40:34.832940+0000","last_became_active":"2026-02-22T22:37:34.543245+0000","last_became_peered":"2026-02-22T22:37:34.543245+0000","last_unstale":"2026-02-22T22:40:34.832940+0000","last_undegraded":"2026-02-22T22:40:34.832940+0000","last_fullsized":"2026-02-22T22:40:34.832940+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:34.832871+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:34.832871+0000","last_clean_scrub_stamp":"2026-02-22T22:40:34.832871+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:23:07.799618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00068412599999999996,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"45'3","reported_seq":88,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:30.900259+0000","last_change":"2026-02-22T22:40:30.900274+0000","last_active":"2026-02-22T22:40:30.900259+0000","last_peered":"2026-02-22T22:40:30.900259+0000","last_clean":"2026-02-22T22:40:30.900259+0000","last_became_active":"2026-02-22T22:37:34.543078+0000","last_became_peered":"2026-02-22T22:37:34.543078+0000","last_unstale":"2026-02-22T22:40:30.900259+0000","last_undegraded":"2026-02-22T22:40:30.900259+0000","last_fullsized":"2026-02-22T22:40:30.900259+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"45'3","last_scrub_stamp":"2026-02-22T22:40:30.900214+0000","last_deep_scrub":"45'3","last_deep_scrub_stamp":"2026-02-22T22:40:30.900214+0000","last_clean_scrub_stamp":"2026-02-22T22:40:30.900214+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:17:09.541456+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00066853699999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":87,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:31.222882+0000","last_change":"2026-02-22T22:40:31.222895+0000","last_active":"2026-02-22T22:40:31.222882+0000","last_peered":"2026-02-22T22:40:31.222882+0000","last_clean":"2026-02-22T22:40:31.222882+0000","last_became_active":"2026-02-22T22:37:34.542908+0000","last_became_peered":"2026-02-22T22:37:34.542908+0000","last_unstale":"2026-02-22T22:40:31.222882+0000","last_undegraded":"2026-02-22T22:40:31.222882+0000","last_fullsized":"2026-02-22T22:40:31.222882+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:31.222831+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:31.222831+0000","last_clean_scrub_stamp":"2026-02-22T22:40:31.222831+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:24:32.101154+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043051299999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":87,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:32.189188+0000","last_change":"2026-02-22T22:40:32.189201+0000","last_active":"2026-02-22T22:40:32.189188+0000","last_peered":"2026-02-22T22:40:32.189188+0000","last_clean":"2026-02-22T22:40:32.189188+0000","last_became_active":"2026-02-22T22:37:34.542961+0000","last_became_peered":"2026-02-22T22:37:34.542961+0000","last_unstale":"2026-02-22T22:40:32.189188+0000","last_undegraded":"2026-02-22T22:40:32.189188+0000","last_fullsized":"2026-02-22T22:40:32.189188+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:40:32.189140+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:40:32.189140+0000","last_clean_scrub_stamp":"2026-02-22T22:40:32.189140+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:17:55.371831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000344994,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"45'2","reported_seq":89,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:31.816520+0000","last_change":"2026-02-22T22:40:31.816533+0000","last_active":"2026-02-22T22:40:31.816520+0000","last_peered":"2026-02-22T22:40:31.816520+0000","last_clean":"2026-02-22T22:40:31.816520+0000","last_became_active":"2026-02-22T22:37:34.542584+0000","last_became_peered":"2026-02-22T22:37:34.542584+0000","last_unstale":"2026-02-22T22:40:31.816520+0000","last_undegraded":"2026-02-22T22:40:31.816520+0000","last_fullsized":"2026-02-22T22:40:31.816520+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"45'2","last_scrub_stamp":"2026-02-22T22:40:31.816486+0000","last_deep_scrub":"45'2","last_deep_scrub_stamp":"2026-02-22T22:40:31.816486+0000","last_clean_scrub_stamp":"2026-02-22T22:40:31.816486+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:08:54.669408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00060424800000000005,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"45'74","reported_seq":180,"reported_epoch":45,"state":"active+clean","last_fresh":"2026-02-22T22:40:30.772662+0000","last_change":"2026-02-22T22:40:30.772686+0000","last_active":"2026-02-22T22:40:30.772662+0000","last_peered":"2026-02-22T22:40:30.772662+0000","last_clean":"2026-02-22T22:40:30.772662+0000","last_became_active":"2026-02-22T22:37:30.530398+0000","last_became_peered":"2026-02-22T22:37:30.530398+0000","last_unstale":"2026-02-22T22:40:30.772662+0000","last_undegraded":"2026-02-22T22:40:30.772662+0000","last_fullsized":"2026-02-22T22:40:30.772662+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"45'74","last_scrub_stamp":"2026-02-22T22:40:30.772609+0000","last_deep_scrub":"45'74","last_deep_scrub_stamp":"2026-02-22T22:40:30.772609+0000","last_clean_scrub_stamp":"2026-02-22T22:40:30.772609+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:14:32.237590+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":0,"up_from":9,"seq":38654705708,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32080,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339760,"statfs":{"total":96636764160,"available":96603914240,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":9,"seq":38654705707,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27336,"kb_used_data":376,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344504,"statfs":{"total":96636764160,"available":96608772096,"internally_reserved":0,"allocated":385024,"data_stored":161986,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27589064},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738410,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32080,"kb_used_data":960,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339760,"statfs":{"total":96636764160,"available":96603914240,"internally_reserved":0,"allocated":983040,"data_stored":752373,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:40:51.061 DEBUG:teuthology.orchestra.run.trial078:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-22T22:40:51.227 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:40:51.227 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:40:51.227 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-22T22:40:51.227 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:51.261 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-22T22:40:51.262 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-22T22:40:51.262 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:51.335 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-22T22:40:51.335 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-22T22:40:51.335 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:51.403 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-22T22:40:51.404 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:40:51.404 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-22T22:40:51.404 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:51.474 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-22T22:40:51.475 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:40:51.475 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-22T22:40:51.475 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:40:51.524 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-22T22:40:51.525 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:40:51.525 DEBUG:teuthology.orchestra.run.trial078:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:40:51.591 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:40:51.591 DEBUG:teuthology.orchestra.run.trial078:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-22T22:40:51.658 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:40:51.659 DEBUG:teuthology.orchestra.run.trial078:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-22T22:40:51.725 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:40:51.726 DEBUG:teuthology.orchestra.run.trial078:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-22T22:40:51.842 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:40:51.842 DEBUG:teuthology.misc:Transferring archived files from trial078:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259/data/cluster1.mon.a.tgz 2026-02-22T22:40:51.842 DEBUG:teuthology.orchestra.run.trial078:> mktemp 2026-02-22T22:40:51.858 INFO:teuthology.orchestra.run.trial078.stdout:/tmp/tmp.ywU4l55hkR 2026-02-22T22:40:51.859 DEBUG:teuthology.orchestra.run.trial078:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.ywU4l55hkR 2026-02-22T22:40:51.950 DEBUG:teuthology.orchestra.run.trial078:> sudo chmod 0666 /tmp/tmp.ywU4l55hkR 2026-02-22T22:40:52.034 DEBUG:teuthology.orchestra.remote:trial078:/tmp/tmp.ywU4l55hkR is 219KB 2026-02-22T22:40:52.126 DEBUG:teuthology.orchestra.run.trial078:> rm -fr /tmp/tmp.ywU4l55hkR 2026-02-22T22:40:52.144 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:40:52.144 DEBUG:teuthology.orchestra.run.trial078:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:40:52.185 DEBUG:teuthology.orchestra.run.trial178:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:40:52.263 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:40:52.278 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:40:52.279 INFO:tasks.ceph:Archiving crash dumps... 2026-02-22T22:40:52.281 DEBUG:teuthology.misc:Transferring archived files from trial078:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259/remote/trial078/crash 2026-02-22T22:40:52.282 DEBUG:teuthology.orchestra.run.trial078:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:40:52.328 DEBUG:teuthology.misc:Transferring archived files from trial178:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259/remote/trial178/crash 2026-02-22T22:40:52.329 DEBUG:teuthology.orchestra.run.trial178:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:40:52.354 INFO:tasks.ceph:Compressing logs... 2026-02-22T22:40:52.354 DEBUG:teuthology.orchestra.run.trial078:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:40:52.373 DEBUG:teuthology.orchestra.run.trial178:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:40:52.410 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45923.log 2026-02-22T22:40:52.410 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42819.log 2026-02-22T22:40:52.410 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.45923.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46013.log 2026-02-22T22:40:52.410 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45923.log.gz 2026-02-22T22:40:52.410 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47550.log 2026-02-22T22:40:52.410 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.42819.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42819.log.gz 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46013.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.37732.log 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46013.log.gz 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47550.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47550.log.gz 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46359.log 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/ceph.tmp-client.admin.37732.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46212.log 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.37732.log.gz 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46359.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46359.log.gz 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46212.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46360.log 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46212.log.gz 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47354.log 2026-02-22T22:40:52.411 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster2-mgr.x.log: /var/log/ceph/cluster1-client.admin.46360.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46360.log.gz 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46628.log 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47354.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47354.log.gz 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43108.log 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46628.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46628.log.gz 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.43108.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43108.log.gz 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46214.log 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47844.log 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46214.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46214.log.gz 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47844.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42634.log 2026-02-22T22:40:52.412 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47844.log.gz 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47697.log 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.42634.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45853.log 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42634.log.gz 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47697.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47648.log 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47697.log.gz 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.45853.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45853.log.gz 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47648.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46508.log 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47648.log.gz 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46797.log 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46508.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46508.log.gz 2026-02-22T22:40:52.413 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47501.log 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr:gzip/var/log/ceph/cluster1-client.admin.46797.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.46361.log 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46797.log.gz 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46748.log 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47501.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47501.log.gz 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46361.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46361.log.gz 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46111.log 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46748.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46748.log.gz 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46625.log 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46111.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46213.log 2026-02-22T22:40:52.414 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46111.log.gz 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46625.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46625.log.gz 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47940.log 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46213.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46062.log 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46213.log.gz 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47940.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47940.log.gz 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46062.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46062.log.gz 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45948.log 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46660.log 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.45948.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47795.log 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.45948.log.gz 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr:gzip/var/log/ceph/cluster1-client.admin.46660.log: -5 --verbose -- /var/log/ceph/cluster1-client.admin.42866.log 2026-02-22T22:40:52.415 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46660.log.gz 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47795.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43040.log 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47795.log.gz 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.42866.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45762.log 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42866.log.gz 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.43040.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47893.log 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43040.log.gz 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.45762.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45762.log.gz 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46507.log 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47893.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47599.log 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47893.log.gz 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46507.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46505.log 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46507.log.gz 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47599.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46846.log 2026-02-22T22:40:52.416 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47599.log.gz 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46505.log: gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46505.log.gz 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.46846.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47746.log 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46846.log.gz 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42913.log 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47746.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47403.log 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47746.log.gz 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr: 89.7% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.42913.log: gzip -5 --verbose 0.0% -- -- replaced with /var/log/ceph/cluster1-client.admin.42913.log.gz /var/log/ceph/cluster1.log 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr: 2026-02-22T22:40:52.417 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1-client.admin.47403.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47452.log 2026-02-22T22:40:52.418 INFO:teuthology.orchestra.run.trial078.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47403.log.gz 2026-02-22T22:40:52.418 INFO:teuthology.orchestra.run.trial078.stderr:/var/log/ceph/cluster1.log: /var/log/ceph/cluster1-client.admin.47452.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47452.log.gz 2026-02-22T22:40:52.418 INFO:teuthology.orchestra.run.trial078.stderr: 85.8% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-22T22:40:52.418 INFO:teuthology.orchestra.run.trial078.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-22T22:40:52.418 INFO:teuthology.orchestra.run.trial078.stderr: 91.6% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-22T22:40:52.424 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50876.log 2026-02-22T22:40:52.425 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50710.log 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50876.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52433.log 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50710.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48321.log 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51648.log 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52433.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52433.log.gz 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50710.log.gz 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48321.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43598.log 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48321.log.gz 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50460.log 2026-02-22T22:40:52.453 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.51648.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.51648.log.gz 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.43598.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43598.log.gz 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50142.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51404.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50460.log: /var/log/ceph/cluster1-client.mirror.50142.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47488.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.51404.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46851.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr: 37.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51404.log.gz 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47488.log: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50142.log.gz 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50295.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47488.log.gz 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.46851.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51270.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46851.log.gz 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50295.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47199.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51270.log: gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50876.log.gz 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47199.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49313.log 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47199.log.gz 2026-02-22T22:40:52.454 INFO:teuthology.orchestra.run.trial178.stderr: 89.4% 90.0% -- replaced with /var/log/ceph/cluster1-client.mirror.50295.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.51270.log.gz 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr: 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51051.log 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr: 85.0% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49313.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49793.log 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr: 57.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49313.log.gz 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.51051.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48608.log 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49793.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51881.log 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51051.log.gz 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.48608.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.49793.log.gz 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48608.log.gz 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51380.log 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51881.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51881.log.gz 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47782.log 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50460.log.gz 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51380.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48997.log 2026-02-22T22:40:52.455 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47782.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49845.log 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.admin.51380.log.gz 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47782.log.gz 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.48997.log: 58.0% -- replaced with /var/log/ceph/cluster2-client.mirror.48997.log.gz 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48929.log 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.49845.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49985.log 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49845.log.gz 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.48929.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51530.log 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr: 55.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48929.log.gz 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.49985.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50761.log 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr: 58.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49985.log.gz 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51530.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49086.log 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51530.log.gz 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50761.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50557.log 2026-02-22T22:40:52.456 INFO:teuthology.orchestra.run.trial178.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50761.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49464.log 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.49086.log: 65.1% -- replaced with /var/log/ceph/cluster2-client.admin.49086.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50557.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46691.log 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50557.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49464.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47197.log 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.46691.log: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49464.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46691.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52384.log 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47197.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47197.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50371.log 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52384.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51203.log 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52384.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50371.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46949.log 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50371.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51203.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43806.log 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.51203.log.gz 2026-02-22T22:40:52.457 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.46949.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48946.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46949.log.gz 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.43806.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43806.log.gz 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52139.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.48946.log: 2.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48946.log.gz 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52139.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50395.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52139.log.gz 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48027.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.50395.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.rbd-mirror-peer.49205.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr: 90.1% -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.50395.log.gz 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.48027.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48027.log.gz 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50899.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48707.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.rbd-mirror-peer.49205.log: /var/log/ceph/cluster2-client.mirror.50899.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48775.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.48707.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50581.log 2026-02-22T22:40:52.458 INFO:teuthology.orchestra.run.trial178.stderr: 53.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48707.log.gz 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50899.log.gz 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.48775.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51422.log 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr: 57.1% -- replaced with /var/log/ceph/cluster1-client.mirror.48775.log.gz 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.50581.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49443.log 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.admin.50581.log.gz 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51422.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49922.log 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49443.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51422.log.gz 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49285.log 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.49922.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51138.log 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.49443.log.gz 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.admin.49922.log.gz 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49285.log: 56.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49285.log.gz 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48690.log 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51138.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48656.log 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.48690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50634.log 2026-02-22T22:40:52.459 INFO:teuthology.orchestra.run.trial178.stderr: 67.4% -- replaced with /var/log/ceph/cluster1-client.admin.48690.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.51138.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.48656.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.rbd-mirror-peer.49160.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.48656.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.50634.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48724.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.rbd-mirror-peer.49160.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.50634.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52188.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.48724.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49637.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr: 2.3% -- replaced with /var/log/ceph/cluster1-client.mirror.48724.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52188.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52188.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51499.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.49637.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51672.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51499.log: 87.4% -- replaced with /var/log/ceph/cluster2-client.admin.49637.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50063.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51499.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52480.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51672.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51672.log.gz 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50063.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50950.log 2026-02-22T22:40:52.460 INFO:teuthology.orchestra.run.trial178.stderr: 83.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50063.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52480.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52480.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.49166.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50950.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47978.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.0.49166.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48826.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.49166.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.47978.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51225.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47978.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48826.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48826.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47052.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51225.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48174.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.51225.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47052.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48792.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47052.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48174.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48174.log.gz 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49068.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.48792.log: gzip 57.4% -5 --verbose -- replaced with /var/log/ceph/cluster1-client.mirror.48792.log.gz -- 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr: /var/log/ceph/cluster2-client.mirror.50223.log 2026-02-22T22:40:52.461 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.49068.log: 63.2% -- replaced with /var/log/ceph/cluster1-client.admin.49068.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51572.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50223.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49031.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr: 92.8% -- replaced with /var/log/ceph/cluster2-client.rbd-mirror-peer.49160.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.50223.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51572.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.49031.log: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.51572.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr: 58.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49031.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49219.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49943.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.49219.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47831.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49219.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr: 92.8% -- replaced with /var/log/ceph/cluster1-client.rbd-mirror-peer.49205.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.49943.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51072.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.admin.49943.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47831.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48963.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47831.log.gz 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51072.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47586.log 2026-02-22T22:40:52.462 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.48963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51167.log 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51072.log.gz 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48963.log.gz 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47586.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51117.log 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47586.log.gz 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.51167.log: gzip 57.0% -5 --verbose -- replaced with /var/log/ceph/cluster2-client.mirror.51167.log.gz -- 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr: /var/log/ceph/cluster2-client.admin.48223.log 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51117.log: gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48223.log: 90.3% 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51117.log.gz -- replaced with /var/log/ceph/cluster2-client.admin.48223.log.gz 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr: 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50346.log 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49014.log 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50346.log: gzip -5 --verbose -- 87.7% /var/log/ceph/cluster2-client.admin.48878.log -- replaced with /var/log/ceph/cluster2-client.mirror.50346.log.gz 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr: 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.49014.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48370.log 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr: 57.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49014.log.gz 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48878.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49118.log 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.48878.log.gz 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48370.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48370.log.gz 2026-02-22T22:40:52.463 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49744.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.49118.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49869.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr: 73.0% -- replaced with /var/log/ceph/cluster2-client.admin.49118.log.gz 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49744.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50183.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.49869.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49511.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.49869.log.gz 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50183.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50974.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr: 83.1% -- replaced with /var/log/ceph/cluster2-client.mirror.50183.log.gz 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49511.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50533.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50974.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52090.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50974.log.gz 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49560.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.50533.log.gz 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52090.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43980.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52090.log.gz 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49560.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49151.log 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49560.log.gz 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.43980.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43980.log.gz 2026-02-22T22:40:52.464 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50103.log 2026-02-22T22:40:52.465 INFO:teuthology.orchestra.run.trial078.stderr: 91.1% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-22T22:40:52.475 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.0.49151.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50273.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr: 19.6% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49151.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50103.log: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50103.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49613.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50273.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47635.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49744.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.49613.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49364.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.49613.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.47635.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47635.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51930.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49364.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51359.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr: 83.2% -- replaced with /var/log/ceph/cluster1-client.mirror.49364.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51930.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51930.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50926.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.mirror.49511.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.51359.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48272.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50926.log: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.51359.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48673.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50926.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50003.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48272.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48272.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.48673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51185.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr: 80.0% -- replaced with /var/log/ceph/cluster1-client.admin.48673.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50003.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43759.log 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr: 83.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50003.log.gz 2026-02-22T22:40:52.476 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.51185.log: gzip -5 --verbose -- 37.4% /var/log/ceph/cluster1-client.admin.51551.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: -- replaced with /var/log/ceph/cluster2-client.mirror.51185.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.43759.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43759.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50082.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.51551.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49766.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50273.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50082.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51551.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50688.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49766.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50787.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 89.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50082.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49664.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.49766.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50787.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49584.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.mirror.50787.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.49664.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49967.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.49664.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.49584.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47292.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 87.7% -- replaced with /var/log/ceph/cluster2-client.mirror.49584.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49967.log: 57.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49967.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47355.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47292.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50998.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47292.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47355.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50422.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47355.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.50998.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51979.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 87.2% -- replaced with /var/log/ceph/cluster1-client.admin.50998.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.50422.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47439.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51979.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.50422.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51979.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47051.log 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47439.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47439.log.gz 2026-02-22T22:40:52.477 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51444.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47051.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47051.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46786.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51444.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48912.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.46786.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.51444.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 80.4% -- replaced with /var/log/ceph/cluster2-client.admin.46786.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47356.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48912.log: 67.4% -- replaced with /var/log/ceph/cluster2-client.admin.48912.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49896.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50688.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47356.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47356.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.invalid.49205.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.49896.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47929.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 87.3% -- replaced with /var/log/ceph/cluster2-client.admin.49896.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.invalid.49205.log: gzip -5 64.7% --verbose -- -- replaced with /var/log/ceph/cluster1-client.invalid.49205.log.gz /var/log/ceph/cluster1-client.mirror.49424.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.47929.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47929.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50655.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49424.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52028.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 83.1% -- replaced with /var/log/ceph/cluster1-client.mirror.49424.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.50655.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49160.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.50655.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52028.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47684.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52028.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.0.49160.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48852.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.47684.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47684.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49817.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48852.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52335.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48852.log.gz 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.49817.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49690.log 2026-02-22T22:40:52.478 INFO:teuthology.orchestra.run.trial178.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.49817.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52335.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52335.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47733.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.49690.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48125.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.admin.49690.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.47733.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47733.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47537.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.48125.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48809.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48125.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47537.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47537.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50202.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.48809.log: 57.1% -- replaced with /var/log/ceph/cluster1-client.mirror.48809.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48741.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50202.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50043.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.48741.log: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48741.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49404.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50043.log: 89.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50202.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50509.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr: 87.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50043.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49404.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.44048.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49404.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50509.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51096.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr: 87.8% -- replaced with /var/log/ceph/cluster2-client.mirror.50509.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.44048.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.44048.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51624.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.51096.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52237.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51624.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.51624.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip 90.3% -5 -- replaced with /var/log/ceph/cluster1-client.admin.51096.log.gz --verbose 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr: -- /var/log/ceph/cluster1-client.mirror.50322.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52237.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52237.log.gz 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51025.log 2026-02-22T22:40:52.479 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50322.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47198.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 87.7% -- replaced with /var/log/ceph/cluster1-client.mirror.50322.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.51025.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47880.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47198.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.51025.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47198.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49383.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47880.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47880.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49711.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49383.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46761.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.49711.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50482.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 89.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49383.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.admin.49711.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.46761.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46761.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49533.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.50482.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43853.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.49533.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48758.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 89.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50482.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.43853.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43853.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46611.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 89.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49533.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50811.log/var/log/ceph/cluster1-client.mirror.48758.log: 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48758.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.46611.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50838.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46611.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.50811.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48632.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 87.2% -- replaced with /var/log/ceph/cluster2-client.admin.50811.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.50838.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48076.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.48632.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.50838.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48632.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49051.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.48076.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48076.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48895.log 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr: 92.3% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49160.log.gz 2026-02-22T22:40:52.480 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.49051.log: 64.8% -- replaced with /var/log/ceph/cluster1-client.admin.49051.log.gzgzip 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.admin.50608.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.48895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50123.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr: 80.1% -- replaced with /var/log/ceph/cluster2-client.admin.48895.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.admin.50608.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50163.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50123.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.50608.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48980.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr: 83.2% -- replaced with /var/log/ceph/cluster1-client.mirror.50123.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50163.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46900.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr: 87.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50163.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.48980.log: 2.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48980.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.38399.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.46900.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46900.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52286.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/ceph.tmp-client.admin.38399.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.38399.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51338.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.52286.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52286.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.51338.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.49205.log 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-osd.2.log: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.51338.log.gz 2026-02-22T22:40:52.481 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50737.log 2026-02-22T22:40:52.482 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.mirror.0.49205.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47050.log 2026-02-22T22:40:52.482 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50737.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50022.log 2026-02-22T22:40:52.482 INFO:teuthology.orchestra.run.trial178.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50737.log.gz 2026-02-22T22:40:52.482 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster2-client.admin.47050.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47050.log.gz 2026-02-22T22:40:52.482 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51600.log 2026-02-22T22:40:52.482 INFO:teuthology.orchestra.run.trial178.stderr:/var/log/ceph/cluster1-client.mirror.50022.log: /var/log/ceph/cluster1-client.admin.51600.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.51600.log.gz 2026-02-22T22:40:52.482 INFO:teuthology.orchestra.run.trial178.stderr: 89.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50022.log.gz 2026-02-22T22:40:52.493 INFO:teuthology.orchestra.run.trial178.stderr: 92.3% -- replaced with /var/log/ceph/cluster2-client.mirror.0.49205.log.gz 2026-02-22T22:40:52.505 INFO:teuthology.orchestra.run.trial178.stderr: 91.1% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-22T22:40:52.606 INFO:teuthology.orchestra.run.trial078.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-22T22:40:52.654 INFO:teuthology.orchestra.run.trial078.stderr: 94.4% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-22T22:40:52.688 INFO:teuthology.orchestra.run.trial178.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-22T22:40:52.688 INFO:teuthology.orchestra.run.trial078.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-22T22:40:52.690 INFO:teuthology.orchestra.run.trial078.stderr: 2026-02-22T22:40:52.690 INFO:teuthology.orchestra.run.trial078.stderr:real 0m0.304s 2026-02-22T22:40:52.690 INFO:teuthology.orchestra.run.trial078.stderr:user 0m0.822s 2026-02-22T22:40:52.690 INFO:teuthology.orchestra.run.trial078.stderr:sys 0m0.064s 2026-02-22T22:40:52.700 INFO:teuthology.orchestra.run.trial178.stderr: 94.4% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-22T22:40:52.735 INFO:teuthology.orchestra.run.trial178.stderr: 94.5% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-22T22:40:52.737 INFO:teuthology.orchestra.run.trial178.stderr: 2026-02-22T22:40:52.737 INFO:teuthology.orchestra.run.trial178.stderr:real 0m0.321s 2026-02-22T22:40:52.737 INFO:teuthology.orchestra.run.trial178.stderr:user 0m0.903s 2026-02-22T22:40:52.737 INFO:teuthology.orchestra.run.trial178.stderr:sys 0m0.124s 2026-02-22T22:40:52.738 INFO:tasks.ceph:Archiving logs... 2026-02-22T22:40:52.738 DEBUG:teuthology.misc:Transferring archived files from trial078:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259/remote/trial078/log 2026-02-22T22:40:52.740 DEBUG:teuthology.orchestra.run.trial078:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:40:52.877 DEBUG:teuthology.misc:Transferring archived files from trial178:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259/remote/trial178/log 2026-02-22T22:40:52.878 DEBUG:teuthology.orchestra.run.trial178:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:40:53.105 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-22T22:40:53.115 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-22T22:40:53.115 DEBUG:teuthology.orchestra.run.trial078:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:40:53.118 DEBUG:teuthology.orchestra.run.trial178:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:40:53.275 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-22T22:40:53.287 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-22T22:40:53.287 DEBUG:teuthology.orchestra.run.trial078:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:40:53.289 DEBUG:teuthology.orchestra.run.trial178:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:40:53.303 INFO:teuthology.orchestra.run.trial078.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:40:53.305 INFO:teuthology.orchestra.run.trial178.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:40:53.324 INFO:teuthology.orchestra.run.trial078.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:40:53.324 INFO:teuthology.orchestra.run.trial078.stdout:=============================================================================== 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial078.stdout:^- 149.28.61.105.vultruserc> 3 6 377 26 -228us[ +416us] +/- 32ms 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial078.stdout:^- ntp.maxhost.io 2 6 127 81 -35us[ +496us] +/- 56ms 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial078.stdout:^- 23-95-35-34-host.colocro> 2 6 377 25 -739us[ -739us] +/- 36ms 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial078.stdout:^* pool-96-231-54-150.washd> 1 6 377 26 +421us[+1065us] +/- 5936us 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial178.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial178.stdout:=============================================================================== 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial178.stdout:^- 149.28.61.105.vultruserc> 3 6 377 24 -796us[ -796us] +/- 32ms 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial178.stdout:^- ntp.maxhost.io 2 6 277 16 -747us[ -747us] +/- 55ms 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial178.stdout:^- 23-95-35-34-host.colocro> 2 6 377 27 -1364us[-1364us] +/- 36ms 2026-02-22T22:40:53.325 INFO:teuthology.orchestra.run.trial178.stdout:^* pool-96-231-54-150.washd> 1 6 177 91 -526us[-1019us] +/- 6254us 2026-02-22T22:40:53.325 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-22T22:40:53.337 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-22T22:40:53.337 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-22T22:40:53.352 DEBUG:teuthology.orchestra.run.trial078:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T22:40:53.420 DEBUG:teuthology.orchestra.run.trial178:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T22:40:53.476 DEBUG:teuthology.orchestra.run.trial078:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:40:53.504 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:40:53.504 DEBUG:teuthology.orchestra.run.trial178:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:40:53.530 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:40:53.531 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-22T22:40:53.548 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-22T22:40:53.563 INFO:teuthology.task.internal:Duration was 464.114143 seconds 2026-02-22T22:40:53.564 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-22T22:40:53.581 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-22T22:40:53.581 DEBUG:teuthology.orchestra.run.trial078:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:40:53.583 DEBUG:teuthology.orchestra.run.trial178:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:40:53.617 INFO:teuthology.orchestra.run.trial078.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:40:53.620 INFO:teuthology.orchestra.run.trial178.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:40:53.846 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-22T22:40:53.846 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial078.front.sepia.ceph.com 2026-02-22T22:40:53.846 DEBUG:teuthology.orchestra.run.trial078:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:40:53.867 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial178.front.sepia.ceph.com 2026-02-22T22:40:53.867 DEBUG:teuthology.orchestra.run.trial178:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:40:53.905 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-22T22:40:53.905 DEBUG:teuthology.orchestra.run.trial078:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:40:53.909 DEBUG:teuthology.orchestra.run.trial178:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:40:54.007 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-22T22:40:54.007 DEBUG:teuthology.orchestra.run.trial078:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:40:54.010 DEBUG:teuthology.orchestra.run.trial178:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:40:54.032 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:40:54.032 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:40:54.034 INFO:teuthology.orchestra.run.trial078.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:40:54.034 INFO:teuthology.orchestra.run.trial078.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:40:54.034 INFO:teuthology.orchestra.run.trial078.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:40:54.035 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:40:54.035 INFO:teuthology.orchestra.run.trial178.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:40:54.035 INFO:teuthology.orchestra.run.trial178.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:40:54.035 INFO:teuthology.orchestra.run.trial178.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:40:54.035 INFO:teuthology.orchestra.run.trial178.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:40:54.043 INFO:teuthology.orchestra.run.trial078.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 81.8% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:40:54.044 INFO:teuthology.orchestra.run.trial178.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:40:54.046 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-22T22:40:54.064 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-22T22:40:54.064 DEBUG:teuthology.orchestra.run.trial078:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:40:54.110 DEBUG:teuthology.orchestra.run.trial178:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:40:54.135 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-22T22:40:54.155 DEBUG:teuthology.orchestra.run.trial078:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:40:54.158 DEBUG:teuthology.orchestra.run.trial178:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:40:54.180 INFO:teuthology.orchestra.run.trial078.stdout:kernel.core_pattern = core 2026-02-22T22:40:54.198 INFO:teuthology.orchestra.run.trial178.stdout:kernel.core_pattern = core 2026-02-22T22:40:54.214 DEBUG:teuthology.orchestra.run.trial078:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:40:54.253 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:40:54.254 DEBUG:teuthology.orchestra.run.trial178:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:40:54.270 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:40:54.270 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-22T22:40:54.288 INFO:teuthology.task.internal:Transferring archived files... 2026-02-22T22:40:54.289 DEBUG:teuthology.misc:Transferring archived files from trial078:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259/remote/trial078 2026-02-22T22:40:54.289 DEBUG:teuthology.orchestra.run.trial078:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:40:54.334 DEBUG:teuthology.misc:Transferring archived files from trial178:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64259/remote/trial178 2026-02-22T22:40:54.334 DEBUG:teuthology.orchestra.run.trial178:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:40:54.370 INFO:teuthology.task.internal:Removing archive directory... 2026-02-22T22:40:54.370 DEBUG:teuthology.orchestra.run.trial078:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:40:54.373 DEBUG:teuthology.orchestra.run.trial178:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:40:54.421 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-22T22:40:54.439 INFO:teuthology.task.internal:Not uploading archives. 2026-02-22T22:40:54.439 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-22T22:40:54.456 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-22T22:40:54.457 DEBUG:teuthology.orchestra.run.trial078:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:40:54.460 DEBUG:teuthology.orchestra.run.trial178:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:40:54.474 INFO:teuthology.orchestra.run.trial078.stdout: 917576 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:40 /home/ubuntu/cephtest 2026-02-22T22:40:54.476 INFO:teuthology.orchestra.run.trial178.stdout: 917576 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:40 /home/ubuntu/cephtest 2026-02-22T22:40:54.478 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-22T22:40:54.497 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-22T22:40:54.537 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-stupid supported-random-distro$/{centos_latest} workloads/rbd-mirror-snapshot-bootstrap-workunit} duration: 464.1141428947449 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-22T22:40:54.537 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:40:54.647 INFO:teuthology.run:pass