2026-02-22T22:28:20.173 INFO:root:teuthology version: 1.2.4.dev9+gfa17720d0 2026-02-22T22:28:20.189 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:28:20.272 INFO:teuthology.run:Config: archive_path: /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242 branch: wip-yuri6-testing-2026-02-22-1629-squid description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-low-osd-mem-target supported-random-distro$/{centos_latest} workloads/rbd-mirror-journal-bootstrap-workunit} email: yweinste@redhat.com first_in_suite: false flavor: default job_id: '64242' kernel: branch: distro kdb: true sha1: distro last_in_suite: false machine_type: trial meta: - desc: run two ceph clusters and install rbd-mirror - desc: configure the permissions for client.mirror - desc: configure the permissions for client.mirror - desc: 2 ceph clusters with 1 mon and 3 osds each - desc: run the rbd_mirror_bootstrap.sh workunit to test the rbd-mirror daemon name: yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial no_nested_subset: false openstack: - volumes: count: 3 size: 30 os_type: centos os_version: 9.stream overrides: admin_socket: branch: wip-yuri6-testing-2026-02-22-1629-squid ceph: conf: client: debug rbd: 20 debug rbd_mirror: 15 log to stderr: false rbd default features: 125 client.mirror.0: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.1: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.2: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.3: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.4: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.5: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid client.mirror.6: admin socket: /var/run/ceph/rbd-mirror.$cluster-$name.asok pid file: /var/run/ceph/rbd-mirror.$cluster-$name.pid global: mon client directed command retry: 5 mon warn on pool no app: false ms inject socket failures: 5000 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: bdev async discard: true bdev enable discard: true bluestore block size: 96636764160 bluestore fsck on mount: true debug bluefs: 1/20 debug bluestore: 1/20 debug ms: 1 debug osd: 20 debug rocksdb: 4/10 mon osd backfillfull_ratio: 0.85 mon osd full ratio: 0.9 mon osd nearfull ratio: 0.8 osd failsafe full ratio: 0.95 osd memory target: 1610612736 osd objectstore: bluestore flavor: default fs: xfs log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) - \(OSD_SLOW_PING_TIME sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 selinux: allowlist: - scontext=system_u:system_r:getty_t:s0 thrashosds: bdev_inject_crash: 2 bdev_inject_crash_probability: 0.5 workunit: branch: wip-yuri6-testing-2026-02-22-1629-squid sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 owner: scheduled_yuriw@soko04.front.sepia.ceph.com priority: 69 repo: https://github.com/ceph/ceph-ci.git roles: - - cluster1.mon.a - cluster1.mgr.x - cluster2.mgr.x - cluster1.osd.0 - cluster1.osd.1 - cluster1.osd.2 - cluster1.client.0 - cluster2.client.0 - - cluster2.mon.a - cluster2.osd.0 - cluster2.osd.1 - cluster2.osd.2 - cluster1.client.mirror - cluster1.client.mirror.0 - cluster1.client.mirror.1 - cluster1.client.mirror.2 - cluster1.client.mirror.3 - cluster1.client.mirror.4 - cluster1.client.mirror.5 - cluster1.client.mirror.6 - cluster2.client.mirror - cluster2.client.mirror.0 - cluster2.client.mirror.1 - cluster2.client.mirror.2 - cluster2.client.mirror.3 - cluster2.client.mirror.4 - cluster2.client.mirror.5 - cluster2.client.mirror.6 seed: 9196 sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 sleep_before_teardown: 0 subset: 111/120000 suite: rbd suite_branch: wip-yuri6-testing-2026-02-22-1629-squid suite_path: /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph-ci.git suite_sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 targets: trial194.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOOwTOfNyp/XTybSmuH0uElIAmBOUI32nWYnTuSOjBlkNOFDVz3wDXzW+QYrxRFsxWd5I5/XoImMeyylNgM7qyE= trial198.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEMJdPpeURyplAGCiXMMqIaEeQc0+Iz1U+I5KwvjiAzM2sOh2fsBPnQ35awYfBsrwVyu/68IPJQpYqM+5LHcQBY= tasks: - install: extra_packages: - rbd-mirror - ceph: cluster: cluster1 - ceph: cluster: cluster2 - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd' - exec: cluster1.client.mirror.0: - sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd' - sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd' - workunit: clients: cluster1.client.mirror: - rbd/rbd_mirror_bootstrap.sh env: CEPH_ARGS: '' MIRROR_POOL_MODE: pool RBD_MIRROR_INSTANCES: '1' RBD_MIRROR_MODE: journal RBD_MIRROR_USE_EXISTING_CLUSTER: '1' teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: fa17720d0088c3ac28e473468bfc79eeaff5cd38 timestamp: 2026-02-22_19:35:38 tube: trial user: yuriw verbose: true worker_log: /home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653 2026-02-22T22:28:20.272 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa; will attempt to use it 2026-02-22T22:28:20.272 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa/tasks 2026-02-22T22:28:20.272 INFO:teuthology.run_tasks:Running task internal.check_packages... 2026-02-22T22:28:20.274 INFO:teuthology.task.internal:Checking packages... 2026-02-22T22:28:20.274 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash 'f90f4eaebb28b619cf76e80cbbe997656da69579' 2026-02-22T22:28:20.274 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2026-02-22T22:28:20.274 INFO:teuthology.packaging:ref: None 2026-02-22T22:28:20.274 INFO:teuthology.packaging:tag: None 2026-02-22T22:28:20.274 INFO:teuthology.packaging:branch: wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:28:20.274 INFO:teuthology.packaging:sha1: f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:28:20.274 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&ref=wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:28:20.390 INFO:teuthology.task.internal:Found packages for ceph version 19.2.3-689.gf90f4eae 2026-02-22T22:28:20.391 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2026-02-22T22:28:20.403 INFO:teuthology.task.internal:no buildpackages task found 2026-02-22T22:28:20.403 INFO:teuthology.run_tasks:Running task internal.save_config... 2026-02-22T22:28:20.416 INFO:teuthology.task.internal:Saving configuration 2026-02-22T22:28:20.430 INFO:teuthology.run_tasks:Running task internal.check_lock... 2026-02-22T22:28:20.443 INFO:teuthology.task.internal.check_lock:Checking locks... 2026-02-22T22:28:20.524 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial194.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 22:24:52.792090', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOOwTOfNyp/XTybSmuH0uElIAmBOUI32nWYnTuSOjBlkNOFDVz3wDXzW+QYrxRFsxWd5I5/XoImMeyylNgM7qyE='} 2026-02-22T22:28:20.579 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'trial198.front.sepia.ceph.com', 'description': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242', 'up': True, 'machine_type': 'trial', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '9.stream', 'arch': 'x86_64', 'locked': True, 'locked_since': '2026-02-22 22:24:52.792668', 'locked_by': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEMJdPpeURyplAGCiXMMqIaEeQc0+Iz1U+I5KwvjiAzM2sOh2fsBPnQ35awYfBsrwVyu/68IPJQpYqM+5LHcQBY='} 2026-02-22T22:28:20.580 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2026-02-22T22:28:20.591 INFO:teuthology.task.internal:roles: ubuntu@trial194.front.sepia.ceph.com - ['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'] 2026-02-22T22:28:20.591 INFO:teuthology.task.internal:roles: ubuntu@trial198.front.sepia.ceph.com - ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6'] 2026-02-22T22:28:20.591 INFO:teuthology.run_tasks:Running task console_log... 2026-02-22T22:28:20.726 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x719ceff151c0>, signals=[15]) 2026-02-22T22:28:20.727 INFO:teuthology.run_tasks:Running task internal.connect... 2026-02-22T22:28:20.737 INFO:teuthology.task.internal:Opening connections... 2026-02-22T22:28:20.738 DEBUG:teuthology.task.internal:connecting to ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:28:20.738 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:28:20.801 DEBUG:teuthology.task.internal:connecting to ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:28:20.802 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial198.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:28:20.863 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2026-02-22T22:28:20.875 DEBUG:teuthology.orchestra.run.trial194:> uname -m 2026-02-22T22:28:20.890 INFO:teuthology.orchestra.run.trial194.stdout:x86_64 2026-02-22T22:28:20.890 DEBUG:teuthology.orchestra.run.trial194:> cat /etc/os-release 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:NAME="CentOS Stream" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:VERSION="9" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:ID="centos" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:ID_LIKE="rhel fedora" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:VERSION_ID="9" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:PLATFORM_ID="platform:el9" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:ANSI_COLOR="0;31" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:LOGO="fedora-logo-icon" 2026-02-22T22:28:20.946 INFO:teuthology.orchestra.run.trial194.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T22:28:20.947 INFO:teuthology.orchestra.run.trial194.stdout:HOME_URL="https://centos.org/" 2026-02-22T22:28:20.947 INFO:teuthology.orchestra.run.trial194.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T22:28:20.947 INFO:teuthology.orchestra.run.trial194.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T22:28:20.947 INFO:teuthology.orchestra.run.trial194.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T22:28:20.947 INFO:teuthology.lock.ops:Updating trial194.front.sepia.ceph.com on lock server 2026-02-22T22:28:21.018 DEBUG:teuthology.orchestra.run.trial198:> uname -m 2026-02-22T22:28:21.033 INFO:teuthology.orchestra.run.trial198.stdout:x86_64 2026-02-22T22:28:21.033 DEBUG:teuthology.orchestra.run.trial198:> cat /etc/os-release 2026-02-22T22:28:21.088 INFO:teuthology.orchestra.run.trial198.stdout:NAME="CentOS Stream" 2026-02-22T22:28:21.088 INFO:teuthology.orchestra.run.trial198.stdout:VERSION="9" 2026-02-22T22:28:21.088 INFO:teuthology.orchestra.run.trial198.stdout:ID="centos" 2026-02-22T22:28:21.088 INFO:teuthology.orchestra.run.trial198.stdout:ID_LIKE="rhel fedora" 2026-02-22T22:28:21.088 INFO:teuthology.orchestra.run.trial198.stdout:VERSION_ID="9" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:PLATFORM_ID="platform:el9" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:PRETTY_NAME="CentOS Stream 9" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:ANSI_COLOR="0;31" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:LOGO="fedora-logo-icon" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:CPE_NAME="cpe:/o:centos:centos:9" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:HOME_URL="https://centos.org/" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:BUG_REPORT_URL="https://issues.redhat.com/" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 9" 2026-02-22T22:28:21.089 INFO:teuthology.orchestra.run.trial198.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2026-02-22T22:28:21.089 INFO:teuthology.lock.ops:Updating trial198.front.sepia.ceph.com on lock server 2026-02-22T22:28:21.165 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2026-02-22T22:28:21.177 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2026-02-22T22:28:21.188 INFO:teuthology.task.internal:Checking for old test directory... 2026-02-22T22:28:21.188 DEBUG:teuthology.orchestra.run.trial194:> test '!' -e /home/ubuntu/cephtest 2026-02-22T22:28:21.190 DEBUG:teuthology.orchestra.run.trial198:> test '!' -e /home/ubuntu/cephtest 2026-02-22T22:28:21.204 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2026-02-22T22:28:21.216 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2026-02-22T22:28:21.216 DEBUG:teuthology.orchestra.run.trial194:> test -z $(ls -A /var/lib/ceph) 2026-02-22T22:28:21.246 DEBUG:teuthology.orchestra.run.trial198:> test -z $(ls -A /var/lib/ceph) 2026-02-22T22:28:21.262 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2026-02-22T22:28:23.805 INFO:teuthology.run_tasks:Running task kernel... 2026-02-22T22:28:23.845 INFO:teuthology.task.kernel:normalize config orig: {'branch': 'distro', 'kdb': True, 'sha1': 'distro'} 2026-02-22T22:28:23.846 INFO:teuthology.task.kernel:config {'cluster1.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'cluster2.mon.a': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, timeout 300 2026-02-22T22:28:23.846 DEBUG:teuthology.orchestra.run.trial194:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T22:28:23.847 DEBUG:teuthology.orchestra.run.trial198:> test -f /run/.containerenv -o -f /.dockerenv 2026-02-22T22:28:23.862 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:28:23.863 DEBUG:teuthology.orchestra.run.trial198:> uname -r 2026-02-22T22:28:23.864 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:28:23.864 DEBUG:teuthology.orchestra.run.trial194:> uname -r 2026-02-22T22:28:23.919 INFO:teuthology.orchestra.run.trial198.stdout:5.14.0-681.el9.x86_64 2026-02-22T22:28:23.919 INFO:teuthology.task.kernel:Running kernel on trial198: 5.14.0-681.el9.x86_64 2026-02-22T22:28:23.919 DEBUG:teuthology.orchestra.run.trial198:> sudo yum install -y kernel 2026-02-22T22:28:23.921 INFO:teuthology.orchestra.run.trial194.stdout:5.14.0-681.el9.x86_64 2026-02-22T22:28:23.922 INFO:teuthology.task.kernel:Running kernel on trial194: 5.14.0-681.el9.x86_64 2026-02-22T22:28:23.922 DEBUG:teuthology.orchestra.run.trial194:> sudo yum install -y kernel 2026-02-22T22:28:25.058 INFO:teuthology.orchestra.run.trial198.stdout:CentOS Stream 9 - BaseOS 12 MB/s | 8.9 MB 00:00 2026-02-22T22:28:25.373 INFO:teuthology.orchestra.run.trial194.stdout:CentOS Stream 9 - BaseOS 8.3 MB/s | 8.9 MB 00:01 2026-02-22T22:28:26.265 INFO:teuthology.orchestra.run.trial194.stdout:CentOS Stream 9 - AppStream 74 MB/s | 27 MB 00:00 2026-02-22T22:28:29.656 INFO:teuthology.orchestra.run.trial194.stdout:CentOS Stream 9 - CRB 7.6 MB/s | 7.8 MB 00:01 2026-02-22T22:28:31.660 INFO:teuthology.orchestra.run.trial194.stdout:CentOS Stream 9 - Extras packages 16 kB/s | 20 kB 00:01 2026-02-22T22:28:32.563 INFO:teuthology.orchestra.run.trial194.stdout:Extra Packages for Enterprise Linux 24 MB/s | 20 MB 00:00 2026-02-22T22:28:35.653 INFO:teuthology.orchestra.run.trial198.stdout:CentOS Stream 9 - AppStream 2.7 MB/s | 27 MB 00:10 2026-02-22T22:28:35.809 INFO:teuthology.orchestra.run.trial194.stdout:lab-extras 1.6 MB/s | 50 kB 00:00 2026-02-22T22:28:36.819 INFO:teuthology.orchestra.run.trial194.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T22:28:36.819 INFO:teuthology.orchestra.run.trial194.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T22:28:36.838 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:28:36.838 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:28:36.838 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:28:36.861 DEBUG:teuthology.orchestra.run.trial194:> echo no | sudo yum reinstall kernel || true 2026-02-22T22:28:37.162 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:28:35 PM UTC. 2026-02-22T22:28:37.242 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:28:37.243 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:28:37.243 INFO:teuthology.orchestra.run.trial194.stdout: Package Architecture Version Repository Size 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout:Reinstalling: 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 803 k 2026-02-22T22:28:37.244 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 0 2026-02-22T22:28:37.245 INFO:teuthology.orchestra.run.trial194.stderr:Operation aborted. 2026-02-22T22:28:37.263 INFO:teuthology.orchestra.run.trial194.stdout:Is this ok [y/N]: 2026-02-22T22:28:37.264 DEBUG:teuthology.orchestra.run.trial194:> sudo yum reinstall -y kernel || true 2026-02-22T22:28:37.564 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:28:35 PM UTC. 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout: Package Architecture Version Repository Size 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:Reinstalling: 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 803 k 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 0 2026-02-22T22:28:37.644 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:28:37.794 INFO:teuthology.orchestra.run.trial194.stdout:kernel-5.14.0-681.el9.x86_64.rpm 10 MB/s | 803 kB 00:00 2026-02-22T22:28:37.795 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:28:37.795 INFO:teuthology.orchestra.run.trial194.stdout:Total 5.2 MB/s | 803 kB 00:00 2026-02-22T22:28:37.805 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:28:37.840 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:28:37.840 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:28:37.844 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:28:37.844 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:28:37.868 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:28:37.872 INFO:teuthology.orchestra.run.trial194.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:28:37.873 INFO:teuthology.orchestra.run.trial194.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:28:37.986 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:28:37.986 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:28:38.040 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:28:38.040 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:28:38.040 INFO:teuthology.orchestra.run.trial194.stdout:Reinstalled: 2026-02-22T22:28:38.040 INFO:teuthology.orchestra.run.trial194.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T22:28:38.040 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:28:38.040 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:28:38.066 DEBUG:teuthology.orchestra.run.trial194:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T22:28:38.092 INFO:teuthology.orchestra.run.trial194.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T22:28:38.093 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T22:28:38.093 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T22:28:38.093 INFO:teuthology.task.kernel:Enabling kdb on cluster1.mon.a... 2026-02-22T22:28:38.093 DEBUG:teuthology.orchestra.run.trial194:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T22:28:38.157 INFO:teuthology.orchestra.run.trial194.stdout:ttyS1 2026-02-22T22:28:38.177 DEBUG:teuthology.parallel:result is None 2026-02-22T22:28:39.682 INFO:teuthology.orchestra.run.trial198.stdout:CentOS Stream 9 - CRB 4.6 MB/s | 7.8 MB 00:01 2026-02-22T22:28:40.893 INFO:teuthology.orchestra.run.trial198.stdout:CentOS Stream 9 - Extras packages 41 kB/s | 20 kB 00:00 2026-02-22T22:28:41.586 INFO:teuthology.orchestra.run.trial198.stdout:Extra Packages for Enterprise Linux 32 MB/s | 20 MB 00:00 2026-02-22T22:28:44.791 INFO:teuthology.orchestra.run.trial198.stdout:lab-extras 2.5 MB/s | 50 kB 00:00 2026-02-22T22:28:45.803 INFO:teuthology.orchestra.run.trial198.stdout:Package kernel-5.14.0-645.el9.x86_64 is already installed. 2026-02-22T22:28:45.803 INFO:teuthology.orchestra.run.trial198.stdout:Package kernel-5.14.0-681.el9.x86_64 is already installed. 2026-02-22T22:28:45.822 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:28:45.823 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:28:45.823 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:28:45.845 DEBUG:teuthology.orchestra.run.trial198:> echo no | sudo yum reinstall kernel || true 2026-02-22T22:28:46.149 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:28:44 PM UTC. 2026-02-22T22:28:46.230 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:28:46.230 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout: Package Architecture Version Repository Size 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout:Reinstalling: 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:28:46.231 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 803 k 2026-02-22T22:28:46.232 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 0 2026-02-22T22:28:46.232 INFO:teuthology.orchestra.run.trial198.stderr:Operation aborted. 2026-02-22T22:28:46.251 INFO:teuthology.orchestra.run.trial198.stdout:Is this ok [y/N]: 2026-02-22T22:28:46.252 DEBUG:teuthology.orchestra.run.trial198:> sudo yum reinstall -y kernel || true 2026-02-22T22:28:46.552 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:02 ago on Sun 22 Feb 2026 10:28:44 PM UTC. 2026-02-22T22:28:46.632 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout: Package Architecture Version Repository Size 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout:Reinstalling: 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout: kernel x86_64 5.14.0-681.el9 baseos 803 k 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:28:46.633 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:28:46.634 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 803 k 2026-02-22T22:28:46.634 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 0 2026-02-22T22:28:46.634 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:28:48.316 INFO:teuthology.orchestra.run.trial198.stdout:kernel-5.14.0-681.el9.x86_64.rpm 563 kB/s | 803 kB 00:01 2026-02-22T22:28:48.316 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:28:48.316 INFO:teuthology.orchestra.run.trial198.stdout:Total 477 kB/s | 803 kB 00:01 2026-02-22T22:28:48.326 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:28:48.360 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:28:48.360 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:28:48.365 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:28:48.365 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:28:48.389 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:28:48.393 INFO:teuthology.orchestra.run.trial198.stdout: Reinstalling : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:28:48.393 INFO:teuthology.orchestra.run.trial198.stdout: Cleanup : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:28:48.503 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:28:48.503 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 1/2 2026-02-22T22:28:48.555 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : kernel-5.14.0-681.el9.x86_64 2/2 2026-02-22T22:28:48.555 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:28:48.555 INFO:teuthology.orchestra.run.trial198.stdout:Reinstalled: 2026-02-22T22:28:48.555 INFO:teuthology.orchestra.run.trial198.stdout: kernel-5.14.0-681.el9.x86_64 2026-02-22T22:28:48.555 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:28:48.555 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:28:48.582 DEBUG:teuthology.orchestra.run.trial198:> rpm -q kernel | sort -rV | head -n 1 2026-02-22T22:28:48.647 INFO:teuthology.orchestra.run.trial198.stdout:kernel-5.14.0-681.el9.x86_64 2026-02-22T22:28:48.648 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 5.14.0-681.el9.x86_64 2026-02-22T22:28:48.648 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2026-02-22T22:28:48.648 INFO:teuthology.task.kernel:Enabling kdb on cluster2.mon.a... 2026-02-22T22:28:48.648 DEBUG:teuthology.orchestra.run.trial198:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2026-02-22T22:28:48.712 INFO:teuthology.orchestra.run.trial198.stdout:ttyS1 2026-02-22T22:28:48.734 DEBUG:teuthology.parallel:result is None 2026-02-22T22:28:48.734 INFO:teuthology.run_tasks:Running task internal.base... 2026-02-22T22:28:48.741 INFO:teuthology.task.internal:Creating test directory... 2026-02-22T22:28:48.741 DEBUG:teuthology.orchestra.run.trial194:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T22:28:48.744 DEBUG:teuthology.orchestra.run.trial198:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2026-02-22T22:28:48.790 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2026-02-22T22:28:48.802 INFO:teuthology.run_tasks:Running task internal.archive... 2026-02-22T22:28:48.816 INFO:teuthology.task.internal:Creating archive directory... 2026-02-22T22:28:48.816 DEBUG:teuthology.orchestra.run.trial194:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T22:28:48.818 DEBUG:teuthology.orchestra.run.trial198:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2026-02-22T22:28:48.858 INFO:teuthology.run_tasks:Running task internal.coredump... 2026-02-22T22:28:48.869 INFO:teuthology.task.internal:Enabling coredump saving... 2026-02-22T22:28:48.870 DEBUG:teuthology.orchestra.run.trial194:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T22:28:48.875 DEBUG:teuthology.orchestra.run.trial198:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2026-02-22T22:28:48.896 INFO:teuthology.orchestra.run.trial194.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:28:48.905 INFO:teuthology.orchestra.run.trial194.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:28:48.911 INFO:teuthology.orchestra.run.trial198.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:28:48.919 INFO:teuthology.orchestra.run.trial198.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2026-02-22T22:28:48.921 INFO:teuthology.run_tasks:Running task internal.sudo... 2026-02-22T22:28:48.933 INFO:teuthology.task.internal:Configuring sudo... 2026-02-22T22:28:48.934 DEBUG:teuthology.orchestra.run.trial194:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T22:28:48.950 DEBUG:teuthology.orchestra.run.trial198:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2026-02-22T22:28:48.985 INFO:teuthology.run_tasks:Running task internal.syslog... 2026-02-22T22:28:49.001 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2026-02-22T22:28:49.001 DEBUG:teuthology.orchestra.run.trial194:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T22:28:49.016 DEBUG:teuthology.orchestra.run.trial198:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2026-02-22T22:28:49.041 DEBUG:teuthology.orchestra.run.trial194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:28:49.189 DEBUG:teuthology.orchestra.run.trial194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:28:49.215 DEBUG:teuthology.orchestra.run.trial194:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:28:51.433 DEBUG:teuthology.orchestra.run.trial194:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:28:51.459 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:28:51.459 DEBUG:teuthology.orchestra.run.trial194:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T22:28:51.524 DEBUG:teuthology.orchestra.run.trial198:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:28:51.758 DEBUG:teuthology.orchestra.run.trial198:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:28:51.783 DEBUG:teuthology.orchestra.run.trial198:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:28:51.941 DEBUG:teuthology.orchestra.run.trial198:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:28:51.966 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:28:51.966 DEBUG:teuthology.orchestra.run.trial198:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2026-02-22T22:28:52.031 DEBUG:teuthology.orchestra.run.trial194:> sudo service rsyslog restart 2026-02-22T22:28:52.034 DEBUG:teuthology.orchestra.run.trial198:> sudo service rsyslog restart 2026-02-22T22:28:52.061 INFO:teuthology.orchestra.run.trial194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:28:52.101 INFO:teuthology.orchestra.run.trial198.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:28:52.489 INFO:teuthology.run_tasks:Running task internal.timer... 2026-02-22T22:28:52.502 INFO:teuthology.task.internal:Starting timer... 2026-02-22T22:28:52.502 INFO:teuthology.run_tasks:Running task pcp... 2026-02-22T22:28:52.523 INFO:teuthology.run_tasks:Running task selinux... 2026-02-22T22:28:52.536 DEBUG:teuthology.task:Applying overrides for task selinux: {'allowlist': ['scontext=system_u:system_r:getty_t:s0']} 2026-02-22T22:28:52.536 DEBUG:teuthology.orchestra.run.trial194:> sudo service auditd rotate 2026-02-22T22:28:52.566 INFO:teuthology.orchestra.run.trial194.stdout:Rotating logs: 2026-02-22T22:28:52.568 DEBUG:teuthology.orchestra.run.trial198:> sudo service auditd rotate 2026-02-22T22:28:52.598 INFO:teuthology.orchestra.run.trial198.stdout:Rotating logs: 2026-02-22T22:28:52.600 DEBUG:teuthology.task.selinux:Getting current SELinux state 2026-02-22T22:28:52.600 DEBUG:teuthology.orchestra.run.trial194:> /usr/sbin/getenforce 2026-02-22T22:28:52.624 INFO:teuthology.orchestra.run.trial194.stdout:Permissive 2026-02-22T22:28:52.624 DEBUG:teuthology.orchestra.run.trial198:> /usr/sbin/getenforce 2026-02-22T22:28:52.655 INFO:teuthology.orchestra.run.trial198.stdout:Permissive 2026-02-22T22:28:52.655 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@trial194.front.sepia.ceph.com': 'permissive', 'ubuntu@trial198.front.sepia.ceph.com': 'permissive'} 2026-02-22T22:28:52.656 DEBUG:teuthology.orchestra.run.trial194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:28:52.687 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:28:52.687 DEBUG:teuthology.orchestra.run.trial198:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:28:52.721 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:28:52.721 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2026-02-22T22:28:52.721 DEBUG:teuthology.orchestra.run.trial194:> sudo /usr/sbin/setenforce permissive 2026-02-22T22:28:52.753 DEBUG:teuthology.orchestra.run.trial198:> sudo /usr/sbin/setenforce permissive 2026-02-22T22:28:52.785 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2026-02-22T22:28:52.801 DEBUG:teuthology.repo_utils:Setting repo remote to https://git.ceph.com/ceph-cm-ansible.git 2026-02-22T22:28:52.804 INFO:teuthology.repo_utils:Fetching git.ceph.com_ceph-cm-ansible_main from origin 2026-02-22T22:28:52.915 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2026-02-22T22:28:52.929 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2026-02-22T22:28:52.930 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit trial194.front.sepia.ceph.com,trial198.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2026-02-22T22:31:36.281 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@trial194.front.sepia.ceph.com'), Remote(name='ubuntu@trial198.front.sepia.ceph.com')] 2026-02-22T22:31:36.283 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial194.front.sepia.ceph.com' 2026-02-22T22:31:36.284 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial194.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:31:36.394 DEBUG:teuthology.orchestra.run.trial194:> true 2026-02-22T22:31:36.469 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial194.front.sepia.ceph.com' 2026-02-22T22:31:36.469 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@trial198.front.sepia.ceph.com' 2026-02-22T22:31:36.470 DEBUG:teuthology.orchestra.connection:{'hostname': 'trial198.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2026-02-22T22:31:36.543 DEBUG:teuthology.orchestra.run.trial198:> true 2026-02-22T22:31:36.620 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@trial198.front.sepia.ceph.com' 2026-02-22T22:31:36.620 INFO:teuthology.run_tasks:Running task clock... 2026-02-22T22:31:36.629 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2026-02-22T22:31:36.629 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T22:31:36.630 DEBUG:teuthology.orchestra.run.trial194:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:31:36.632 INFO:teuthology.orchestra.run:Running command with timeout 360 2026-02-22T22:31:36.632 DEBUG:teuthology.orchestra.run.trial198:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:31:36.658 INFO:teuthology.orchestra.run.trial194.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T22:31:36.670 INFO:teuthology.orchestra.run.trial194.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T22:31:36.688 INFO:teuthology.orchestra.run.trial198.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2026-02-22T22:31:36.690 INFO:teuthology.orchestra.run.trial194.stderr:sudo: ntpd: command not found 2026-02-22T22:31:36.699 INFO:teuthology.orchestra.run.trial194.stdout:506 Cannot talk to daemon 2026-02-22T22:31:36.700 INFO:teuthology.orchestra.run.trial198.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2026-02-22T22:31:36.710 INFO:teuthology.orchestra.run.trial194.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T22:31:36.720 INFO:teuthology.orchestra.run.trial194.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T22:31:36.721 INFO:teuthology.orchestra.run.trial198.stderr:sudo: ntpd: command not found 2026-02-22T22:31:36.731 INFO:teuthology.orchestra.run.trial198.stdout:506 Cannot talk to daemon 2026-02-22T22:31:36.743 INFO:teuthology.orchestra.run.trial198.stderr:Failed to start ntp.service: Unit ntp.service not found. 2026-02-22T22:31:36.754 INFO:teuthology.orchestra.run.trial198.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2026-02-22T22:31:36.765 INFO:teuthology.orchestra.run.trial194.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:31:36.783 INFO:teuthology.orchestra.run.trial194.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:31:36.783 INFO:teuthology.orchestra.run.trial194.stdout:=============================================================================== 2026-02-22T22:31:36.783 INFO:teuthology.orchestra.run.trial194.stdout:^? nyc3.us.ntp.li 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:31:36.783 INFO:teuthology.orchestra.run.trial194.stdout:^? chi2.us.ntp.li 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:31:36.783 INFO:teuthology.orchestra.run.trial194.stdout:^? ip216-250-115-174.pbiaas> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:31:36.783 INFO:teuthology.orchestra.run.trial194.stdout:^? time.ecansol.net 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:31:36.799 INFO:teuthology.orchestra.run.trial198.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:31:36.819 INFO:teuthology.orchestra.run.trial198.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:31:36.819 INFO:teuthology.orchestra.run.trial198.stdout:=============================================================================== 2026-02-22T22:31:36.819 INFO:teuthology.orchestra.run.trial198.stdout:^? 69-10-208-170.rainiercon> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:31:36.819 INFO:teuthology.orchestra.run.trial198.stdout:^? ntp2.wiktel.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:31:36.819 INFO:teuthology.orchestra.run.trial198.stdout:^? 172-104-28-175.ip.linode> 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:31:36.820 INFO:teuthology.orchestra.run.trial198.stdout:^? wdc1.us.ntp.li 0 6 0 - +0ns[ +0ns] +/- 0ns 2026-02-22T22:31:36.821 INFO:teuthology.run_tasks:Running task install... 2026-02-22T22:31:36.835 DEBUG:teuthology.task.install:project ceph 2026-02-22T22:31:36.836 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}} 2026-02-22T22:31:36.836 DEBUG:teuthology.task.install:config {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'} 2026-02-22T22:31:36.836 INFO:teuthology.task.install:Using flavor: default 2026-02-22T22:31:36.846 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'ceph-volume', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'ceph-volume', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2026-02-22T22:31:36.846 INFO:teuthology.task.install:extra packages: ['rbd-mirror'] 2026-02-22T22:31:36.847 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T22:31:36.847 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:31:36.850 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': None, 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': [], 'extra_packages': ['rbd-mirror'], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'tag': None, 'wait_for_package': False} 2026-02-22T22:31:36.850 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:31:37.036 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T22:31:37.036 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T22:31:37.049 INFO:teuthology.task.install.rpm:Pulling from https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/ 2026-02-22T22:31:37.049 INFO:teuthology.task.install.rpm:Package version is 19.2.3-689.gf90f4eae 2026-02-22T22:31:37.210 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T22:31:37.211 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:31:37.211 DEBUG:teuthology.orchestra.run.trial194:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T22:31:37.223 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://2.chacra.ceph.com/r/ceph/wip-yuri6-testing-2026-02-22-1629-squid/f90f4eaebb28b619cf76e80cbbe997656da69579/centos/9/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2026-02-22T22:31:37.223 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:31:37.223 DEBUG:teuthology.orchestra.run.trial198:> sudo dd of=/etc/yum.repos.d/ceph.repo 2026-02-22T22:31:37.235 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T22:31:37.235 DEBUG:teuthology.orchestra.run.trial194:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T22:31:37.249 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, ceph-volume, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd, rbd-mirror on remote rpm x86_64 2026-02-22T22:31:37.249 DEBUG:teuthology.orchestra.run.trial198:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;sha1/f90f4eaebb28b619cf76e80cbbe997656da69579/;g' /etc/yum.repos.d/ceph.repo ; fi 2026-02-22T22:31:37.300 DEBUG:teuthology.orchestra.run.trial194:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T22:31:37.314 DEBUG:teuthology.orchestra.run.trial198:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2026-02-22T22:31:37.374 DEBUG:teuthology.orchestra.run.trial194:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T22:31:37.387 DEBUG:teuthology.orchestra.run.trial198:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2026-02-22T22:31:37.437 INFO:teuthology.orchestra.run.trial194.stdout:check_obsoletes = 1 2026-02-22T22:31:37.439 DEBUG:teuthology.orchestra.run.trial194:> sudo yum clean all 2026-02-22T22:31:37.452 INFO:teuthology.orchestra.run.trial198.stdout:check_obsoletes = 1 2026-02-22T22:31:37.453 DEBUG:teuthology.orchestra.run.trial198:> sudo yum clean all 2026-02-22T22:31:37.595 INFO:teuthology.orchestra.run.trial194.stdout:41 files removed 2026-02-22T22:31:37.610 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-radosgw 2026-02-22T22:31:37.611 INFO:teuthology.orchestra.run.trial198.stdout:41 files removed 2026-02-22T22:31:37.626 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-radosgw 2026-02-22T22:31:37.885 INFO:teuthology.orchestra.run.trial194.stdout:ceph packages for x86_64 647 kB/s | 86 kB 00:00 2026-02-22T22:31:37.899 INFO:teuthology.orchestra.run.trial198.stdout:ceph packages for x86_64 663 kB/s | 86 kB 00:00 2026-02-22T22:31:38.017 INFO:teuthology.orchestra.run.trial194.stdout:ceph noarch packages 112 kB/s | 12 kB 00:00 2026-02-22T22:31:38.031 INFO:teuthology.orchestra.run.trial198.stdout:ceph noarch packages 112 kB/s | 12 kB 00:00 2026-02-22T22:31:38.143 INFO:teuthology.orchestra.run.trial194.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T22:31:38.158 INFO:teuthology.orchestra.run.trial198.stdout:ceph source packages 18 kB/s | 1.9 kB 00:00 2026-02-22T22:31:39.179 INFO:teuthology.orchestra.run.trial198.stdout:CentOS Stream 9 - BaseOS 8.9 MB/s | 8.9 MB 00:01 2026-02-22T22:31:39.188 INFO:teuthology.orchestra.run.trial194.stdout:CentOS Stream 9 - BaseOS 8.7 MB/s | 8.9 MB 00:01 2026-02-22T22:31:42.467 INFO:teuthology.orchestra.run.trial198.stdout:CentOS Stream 9 - AppStream 9.7 MB/s | 27 MB 00:02 2026-02-22T22:31:42.917 INFO:teuthology.orchestra.run.trial194.stdout:CentOS Stream 9 - AppStream 8.4 MB/s | 27 MB 00:03 2026-02-22T22:31:45.963 INFO:teuthology.orchestra.run.trial194.stdout:CentOS Stream 9 - CRB 12 MB/s | 7.8 MB 00:00 2026-02-22T22:31:46.074 INFO:teuthology.orchestra.run.trial198.stdout:CentOS Stream 9 - CRB 6.5 MB/s | 7.8 MB 00:01 2026-02-22T22:31:47.170 INFO:teuthology.orchestra.run.trial194.stdout:CentOS Stream 9 - Extras packages 42 kB/s | 20 kB 00:00 2026-02-22T22:31:47.265 INFO:teuthology.orchestra.run.trial198.stdout:CentOS Stream 9 - Extras packages 43 kB/s | 20 kB 00:00 2026-02-22T22:31:47.957 INFO:teuthology.orchestra.run.trial194.stdout:Extra Packages for Enterprise Linux 28 MB/s | 20 MB 00:00 2026-02-22T22:31:48.093 INFO:teuthology.orchestra.run.trial198.stdout:Extra Packages for Enterprise Linux 27 MB/s | 20 MB 00:00 2026-02-22T22:31:51.159 INFO:teuthology.orchestra.run.trial194.stdout:lab-extras 1.3 MB/s | 50 kB 00:00 2026-02-22T22:31:51.313 INFO:teuthology.orchestra.run.trial198.stdout:lab-extras 2.6 MB/s | 50 kB 00:00 2026-02-22T22:31:52.360 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: Package Arch Version Repository Size 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout:Upgrading: 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout:Installing dependencies: 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T22:31:52.361 INFO:teuthology.orchestra.run.trial194.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T22:31:52.362 INFO:teuthology.orchestra.run.trial194.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout:Installing weak dependencies: 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout:Install 33 Packages 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout:Upgrade 2 Packages 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 63 M 2026-02-22T22:31:52.363 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:31:52.515 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:31:52.516 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:31:52.517 INFO:teuthology.orchestra.run.trial198.stdout: Package Arch Version Repository Size 2026-02-22T22:31:52.517 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:31:52.517 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:31:52.517 INFO:teuthology.orchestra.run.trial198.stdout: ceph-radosgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 11 M 2026-02-22T22:31:52.517 INFO:teuthology.orchestra.run.trial198.stdout:Upgrading: 2026-02-22T22:31:52.517 INFO:teuthology.orchestra.run.trial198.stdout: librados2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.4 M 2026-02-22T22:31:52.517 INFO:teuthology.orchestra.run.trial198.stdout: librbd1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.2 M 2026-02-22T22:31:52.518 INFO:teuthology.orchestra.run.trial198.stdout:Installing dependencies: 2026-02-22T22:31:52.518 INFO:teuthology.orchestra.run.trial198.stdout: boost-program-options x86_64 1.75.0-13.el9 appstream 104 k 2026-02-22T22:31:52.518 INFO:teuthology.orchestra.run.trial198.stdout: ceph-base x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.5 M 2026-02-22T22:31:52.518 INFO:teuthology.orchestra.run.trial198.stdout: ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 22 M 2026-02-22T22:31:52.518 INFO:teuthology.orchestra.run.trial198.stdout: ceph-selinux x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 25 k 2026-02-22T22:31:52.518 INFO:teuthology.orchestra.run.trial198.stdout: gperftools-libs x86_64 2.9.1-3.el9 epel 308 k 2026-02-22T22:31:52.519 INFO:teuthology.orchestra.run.trial198.stdout: libarrow x86_64 9.0.0-15.el9 epel 4.4 M 2026-02-22T22:31:52.519 INFO:teuthology.orchestra.run.trial198.stdout: libarrow-doc noarch 9.0.0-15.el9 epel 25 k 2026-02-22T22:31:52.519 INFO:teuthology.orchestra.run.trial198.stdout: libcephfs2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 999 k 2026-02-22T22:31:52.519 INFO:teuthology.orchestra.run.trial198.stdout: libnbd x86_64 1.20.3-4.el9 appstream 164 k 2026-02-22T22:31:52.519 INFO:teuthology.orchestra.run.trial198.stdout: liboath x86_64 2.6.12-1.el9 epel 49 k 2026-02-22T22:31:52.519 INFO:teuthology.orchestra.run.trial198.stdout: libpmemobj x86_64 1.12.1-1.el9 appstream 160 k 2026-02-22T22:31:52.520 INFO:teuthology.orchestra.run.trial198.stdout: librabbitmq x86_64 0.11.0-7.el9 appstream 45 k 2026-02-22T22:31:52.520 INFO:teuthology.orchestra.run.trial198.stdout: libradosstriper1 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 503 k 2026-02-22T22:31:52.520 INFO:teuthology.orchestra.run.trial198.stdout: librdkafka x86_64 1.6.1-102.el9 appstream 662 k 2026-02-22T22:31:52.520 INFO:teuthology.orchestra.run.trial198.stdout: librgw2 x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 5.4 M 2026-02-22T22:31:52.520 INFO:teuthology.orchestra.run.trial198.stdout: libunwind x86_64 1.6.2-1.el9 epel 67 k 2026-02-22T22:31:52.520 INFO:teuthology.orchestra.run.trial198.stdout: lttng-ust x86_64 2.12.0-6.el9 appstream 292 k 2026-02-22T22:31:52.521 INFO:teuthology.orchestra.run.trial198.stdout: mailcap noarch 2.1.49-5.el9 baseos 33 k 2026-02-22T22:31:52.521 INFO:teuthology.orchestra.run.trial198.stdout: parquet-libs x86_64 9.0.0-15.el9 epel 838 k 2026-02-22T22:31:52.521 INFO:teuthology.orchestra.run.trial198.stdout: pciutils x86_64 3.7.0-7.el9 baseos 93 k 2026-02-22T22:31:52.521 INFO:teuthology.orchestra.run.trial198.stdout: python3-ceph-argparse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 45 k 2026-02-22T22:31:52.521 INFO:teuthology.orchestra.run.trial198.stdout: python3-ceph-common x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 142 k 2026-02-22T22:31:52.521 INFO:teuthology.orchestra.run.trial198.stdout: python3-cephfs x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 165 k 2026-02-22T22:31:52.521 INFO:teuthology.orchestra.run.trial198.stdout: python3-rados x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 322 k 2026-02-22T22:31:52.522 INFO:teuthology.orchestra.run.trial198.stdout: python3-rbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 303 k 2026-02-22T22:31:52.522 INFO:teuthology.orchestra.run.trial198.stdout: python3-rgw x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 100 k 2026-02-22T22:31:52.522 INFO:teuthology.orchestra.run.trial198.stdout: qatlib x86_64 25.08.0-2.el9 appstream 240 k 2026-02-22T22:31:52.522 INFO:teuthology.orchestra.run.trial198.stdout: qatzip-libs x86_64 1.3.1-1.el9 appstream 66 k 2026-02-22T22:31:52.522 INFO:teuthology.orchestra.run.trial198.stdout: re2 x86_64 1:20211101-20.el9 epel 191 k 2026-02-22T22:31:52.522 INFO:teuthology.orchestra.run.trial198.stdout: thrift x86_64 0.15.0-4.el9 epel 1.6 M 2026-02-22T22:31:52.523 INFO:teuthology.orchestra.run.trial198.stdout:Installing weak dependencies: 2026-02-22T22:31:52.523 INFO:teuthology.orchestra.run.trial198.stdout: nvme-cli x86_64 2.16-1.el9 baseos 1.2 M 2026-02-22T22:31:52.523 INFO:teuthology.orchestra.run.trial198.stdout: qatlib-service x86_64 25.08.0-2.el9 appstream 37 k 2026-02-22T22:31:52.523 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:31:52.523 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:31:52.523 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:31:52.524 INFO:teuthology.orchestra.run.trial198.stdout:Install 33 Packages 2026-02-22T22:31:52.524 INFO:teuthology.orchestra.run.trial198.stdout:Upgrade 2 Packages 2026-02-22T22:31:52.524 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:31:52.525 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 63 M 2026-02-22T22:31:52.525 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:31:52.853 INFO:teuthology.orchestra.run.trial194.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 19 MB/s | 5.5 MB 00:00 2026-02-22T22:31:52.888 INFO:teuthology.orchestra.run.trial194.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 723 kB/s | 25 kB 00:00 2026-02-22T22:31:52.943 INFO:teuthology.orchestra.run.trial194.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 18 MB/s | 999 kB 00:00 2026-02-22T22:31:52.990 INFO:teuthology.orchestra.run.trial194.stdout:(4/35): libradosstriper1-19.2.3-689.gf90f4eae.e 10 MB/s | 503 kB 00:00 2026-02-22T22:31:53.271 INFO:teuthology.orchestra.run.trial194.stdout:(5/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9.x 15 MB/s | 11 MB 00:00 2026-02-22T22:31:53.291 INFO:teuthology.orchestra.run.trial194.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 2.2 MB/s | 45 kB 00:00 2026-02-22T22:31:53.314 INFO:teuthology.orchestra.run.trial194.stdout:(7/35): python3-ceph-common-19.2.3-689.gf90f4ea 6.2 MB/s | 142 kB 00:00 2026-02-22T22:31:53.341 INFO:teuthology.orchestra.run.trial194.stdout:(8/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 6.1 MB/s | 165 kB 00:00 2026-02-22T22:31:53.372 INFO:teuthology.orchestra.run.trial194.stdout:(9/35): python3-rados-19.2.3-689.gf90f4eae.el9. 10 MB/s | 322 kB 00:00 2026-02-22T22:31:53.378 INFO:teuthology.orchestra.run.trial194.stdout:(10/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_6 14 MB/s | 5.4 MB 00:00 2026-02-22T22:31:53.403 INFO:teuthology.orchestra.run.trial194.stdout:(11/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 9.6 MB/s | 303 kB 00:00 2026-02-22T22:31:53.412 INFO:teuthology.orchestra.run.trial194.stdout:(12/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 2.9 MB/s | 100 kB 00:00 2026-02-22T22:31:53.471 INFO:teuthology.orchestra.run.trial194.stdout:(13/35): mailcap-2.1.49-5.el9.noarch.rpm 486 kB/s | 33 kB 00:00 2026-02-22T22:31:53.500 INFO:teuthology.orchestra.run.trial194.stdout:(14/35): ceph-common-19.2.3-689.gf90f4eae.el9.x 23 MB/s | 22 MB 00:00 2026-02-22T22:31:53.521 INFO:teuthology.orchestra.run.trial194.stdout:(15/35): pciutils-3.7.0-7.el9.x86_64.rpm 1.8 MB/s | 93 kB 00:00 2026-02-22T22:31:53.584 INFO:teuthology.orchestra.run.trial194.stdout:(16/35): nvme-cli-2.16-1.el9.x86_64.rpm 6.7 MB/s | 1.2 MB 00:00 2026-02-22T22:31:53.648 INFO:teuthology.orchestra.run.trial194.stdout:(17/35): libnbd-1.20.3-4.el9.x86_64.rpm 1.3 MB/s | 164 kB 00:00 2026-02-22T22:31:53.677 INFO:teuthology.orchestra.run.trial194.stdout:(18/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 1.7 MB/s | 160 kB 00:00 2026-02-22T22:31:53.692 INFO:teuthology.orchestra.run.trial194.stdout:(19/35): boost-program-options-1.75.0-13.el9.x8 542 kB/s | 104 kB 00:00 2026-02-22T22:31:53.701 INFO:teuthology.orchestra.run.trial198.stdout:(1/35): ceph-base-19.2.3-689.gf90f4eae.el9.x86_ 8.1 MB/s | 5.5 MB 00:00 2026-02-22T22:31:53.714 INFO:teuthology.orchestra.run.trial198.stdout:(2/35): ceph-selinux-19.2.3-689.gf90f4eae.el9.x 1.8 MB/s | 25 kB 00:00 2026-02-22T22:31:53.736 INFO:teuthology.orchestra.run.trial194.stdout:(20/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 519 kB/s | 45 kB 00:00 2026-02-22T22:31:53.776 INFO:teuthology.orchestra.run.trial198.stdout:(3/35): libcephfs2-19.2.3-689.gf90f4eae.el9.x86 16 MB/s | 999 kB 00:00 2026-02-22T22:31:53.819 INFO:teuthology.orchestra.run.trial198.stdout:(4/35): libradosstriper1-19.2.3-689.gf90f4eae.e 11 MB/s | 503 kB 00:00 2026-02-22T22:31:53.837 INFO:teuthology.orchestra.run.trial194.stdout:(21/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 2.0 MB/s | 292 kB 00:00 2026-02-22T22:31:53.858 INFO:teuthology.orchestra.run.trial194.stdout:(22/35): librdkafka-1.6.1-102.el9.x86_64.rpm 3.6 MB/s | 662 kB 00:00 2026-02-22T22:31:53.864 INFO:teuthology.orchestra.run.trial194.stdout:(23/35): qatlib-25.08.0-2.el9.x86_64.rpm 1.8 MB/s | 240 kB 00:00 2026-02-22T22:31:53.910 INFO:teuthology.orchestra.run.trial194.stdout:(24/35): qatlib-service-25.08.0-2.el9.x86_64.rp 512 kB/s | 37 kB 00:00 2026-02-22T22:31:53.931 INFO:teuthology.orchestra.run.trial194.stdout:(25/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 915 kB/s | 66 kB 00:00 2026-02-22T22:31:53.985 INFO:teuthology.orchestra.run.trial198.stdout:(5/35): ceph-radosgw-19.2.3-689.gf90f4eae.el9.x 11 MB/s | 11 MB 00:00 2026-02-22T22:31:54.004 INFO:teuthology.orchestra.run.trial198.stdout:(6/35): python3-ceph-argparse-19.2.3-689.gf90f4 2.4 MB/s | 45 kB 00:00 2026-02-22T22:31:54.030 INFO:teuthology.orchestra.run.trial198.stdout:(7/35): python3-ceph-common-19.2.3-689.gf90f4ea 5.3 MB/s | 142 kB 00:00 2026-02-22T22:31:54.058 INFO:teuthology.orchestra.run.trial198.stdout:(8/35): python3-cephfs-19.2.3-689.gf90f4eae.el9 5.8 MB/s | 165 kB 00:00 2026-02-22T22:31:54.084 INFO:teuthology.orchestra.run.trial198.stdout:(9/35): librgw2-19.2.3-689.gf90f4eae.el9.x86_64 20 MB/s | 5.4 MB 00:00 2026-02-22T22:31:54.088 INFO:teuthology.orchestra.run.trial198.stdout:(10/35): python3-rados-19.2.3-689.gf90f4eae.el9 11 MB/s | 322 kB 00:00 2026-02-22T22:31:54.108 INFO:teuthology.orchestra.run.trial198.stdout:(11/35): python3-rbd-19.2.3-689.gf90f4eae.el9.x 13 MB/s | 303 kB 00:00 2026-02-22T22:31:54.110 INFO:teuthology.orchestra.run.trial198.stdout:(12/35): python3-rgw-19.2.3-689.gf90f4eae.el9.x 4.4 MB/s | 100 kB 00:00 2026-02-22T22:31:54.111 INFO:teuthology.orchestra.run.trial194.stdout:(26/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 138 kB/s | 25 kB 00:00 2026-02-22T22:31:54.192 INFO:teuthology.orchestra.run.trial194.stdout:(27/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 939 kB/s | 308 kB 00:00 2026-02-22T22:31:54.196 INFO:teuthology.orchestra.run.trial194.stdout:(28/35): liboath-2.6.12-1.el9.x86_64.rpm 573 kB/s | 49 kB 00:00 2026-02-22T22:31:54.240 INFO:teuthology.orchestra.run.trial194.stdout:(29/35): libunwind-1.6.2-1.el9.x86_64.rpm 1.4 MB/s | 67 kB 00:00 2026-02-22T22:31:54.241 INFO:teuthology.orchestra.run.trial198.stdout:(13/35): ceph-common-19.2.3-689.gf90f4eae.el9.x 18 MB/s | 22 MB 00:01 2026-02-22T22:31:54.300 INFO:teuthology.orchestra.run.trial194.stdout:(30/35): re2-20211101-20.el9.x86_64.rpm 3.1 MB/s | 191 kB 00:00 2026-02-22T22:31:54.394 INFO:teuthology.orchestra.run.trial194.stdout:(31/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 4.1 MB/s | 838 kB 00:00 2026-02-22T22:31:54.400 INFO:teuthology.orchestra.run.trial194.stdout:(32/35): libarrow-9.0.0-15.el9.x86_64.rpm 9.0 MB/s | 4.4 MB 00:00 2026-02-22T22:31:54.456 INFO:teuthology.orchestra.run.trial194.stdout:(33/35): thrift-0.15.0-4.el9.x86_64.rpm 10 MB/s | 1.6 MB 00:00 2026-02-22T22:31:54.527 INFO:teuthology.orchestra.run.trial194.stdout:(34/35): librados2-19.2.3-689.gf90f4eae.el9.x86 26 MB/s | 3.4 MB 00:00 2026-02-22T22:31:54.531 INFO:teuthology.orchestra.run.trial194.stdout:(35/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 24 MB/s | 3.2 MB 00:00 2026-02-22T22:31:54.534 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:31:54.534 INFO:teuthology.orchestra.run.trial194.stdout:Total 29 MB/s | 63 MB 00:02 2026-02-22T22:31:54.631 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:31:54.650 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:31:54.650 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:31:54.667 INFO:teuthology.orchestra.run.trial198.stdout:(14/35): mailcap-2.1.49-5.el9.noarch.rpm 59 kB/s | 33 kB 00:00 2026-02-22T22:31:54.714 INFO:teuthology.orchestra.run.trial198.stdout:(15/35): boost-program-options-1.75.0-13.el9.x8 2.2 MB/s | 104 kB 00:00 2026-02-22T22:31:54.733 INFO:teuthology.orchestra.run.trial198.stdout:(16/35): libnbd-1.20.3-4.el9.x86_64.rpm 8.8 MB/s | 164 kB 00:00 2026-02-22T22:31:54.745 INFO:teuthology.orchestra.run.trial198.stdout:(17/35): libpmemobj-1.12.1-1.el9.x86_64.rpm 13 MB/s | 160 kB 00:00 2026-02-22T22:31:54.754 INFO:teuthology.orchestra.run.trial198.stdout:(18/35): librabbitmq-0.11.0-7.el9.x86_64.rpm 4.8 MB/s | 45 kB 00:00 2026-02-22T22:31:54.777 INFO:teuthology.orchestra.run.trial198.stdout:(19/35): librdkafka-1.6.1-102.el9.x86_64.rpm 29 MB/s | 662 kB 00:00 2026-02-22T22:31:54.788 INFO:teuthology.orchestra.run.trial198.stdout:(20/35): lttng-ust-2.12.0-6.el9.x86_64.rpm 26 MB/s | 292 kB 00:00 2026-02-22T22:31:54.799 INFO:teuthology.orchestra.run.trial198.stdout:(21/35): qatlib-25.08.0-2.el9.x86_64.rpm 22 MB/s | 240 kB 00:00 2026-02-22T22:31:54.808 INFO:teuthology.orchestra.run.trial198.stdout:(22/35): qatlib-service-25.08.0-2.el9.x86_64.rp 4.0 MB/s | 37 kB 00:00 2026-02-22T22:31:54.818 INFO:teuthology.orchestra.run.trial198.stdout:(23/35): qatzip-libs-1.3.1-1.el9.x86_64.rpm 6.9 MB/s | 66 kB 00:00 2026-02-22T22:31:54.904 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:31:54.905 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:31:54.950 INFO:teuthology.orchestra.run.trial198.stdout:(24/35): pciutils-3.7.0-7.el9.x86_64.rpm 132 kB/s | 93 kB 00:00 2026-02-22T22:31:55.168 INFO:teuthology.orchestra.run.trial198.stdout:(25/35): gperftools-libs-2.9.1-3.el9.x86_64.rpm 879 kB/s | 308 kB 00:00 2026-02-22T22:31:55.196 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:31:55.205 INFO:teuthology.orchestra.run.trial194.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T22:31:55.208 INFO:teuthology.orchestra.run.trial194.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T22:31:55.220 INFO:teuthology.orchestra.run.trial194.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T22:31:55.222 INFO:teuthology.orchestra.run.trial198.stdout:(26/35): libarrow-doc-9.0.0-15.el9.noarch.rpm 462 kB/s | 25 kB 00:00 2026-02-22T22:31:55.278 INFO:teuthology.orchestra.run.trial198.stdout:(27/35): liboath-2.6.12-1.el9.x86_64.rpm 869 kB/s | 49 kB 00:00 2026-02-22T22:31:55.336 INFO:teuthology.orchestra.run.trial198.stdout:(28/35): libunwind-1.6.2-1.el9.x86_64.rpm 1.1 MB/s | 67 kB 00:00 2026-02-22T22:31:55.369 INFO:teuthology.orchestra.run.trial194.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T22:31:55.374 INFO:teuthology.orchestra.run.trial194.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:31:55.404 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:31:55.447 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:31:55.448 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:31:55.460 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:31:55.465 INFO:teuthology.orchestra.run.trial194.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T22:31:55.474 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T22:31:55.483 INFO:teuthology.orchestra.run.trial198.stdout:(29/35): parquet-libs-9.0.0-15.el9.x86_64.rpm 5.6 MB/s | 838 kB 00:00 2026-02-22T22:31:55.496 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:31:55.497 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:31:55.505 INFO:teuthology.orchestra.run.trial198.stdout:(30/35): libarrow-9.0.0-15.el9.x86_64.rpm 7.9 MB/s | 4.4 MB 00:00 2026-02-22T22:31:55.510 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:31:55.540 INFO:teuthology.orchestra.run.trial194.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T22:31:55.541 INFO:teuthology.orchestra.run.trial198.stdout:(31/35): re2-20211101-20.el9.x86_64.rpm 3.2 MB/s | 191 kB 00:00 2026-02-22T22:31:55.546 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T22:31:55.552 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T22:31:55.554 INFO:teuthology.orchestra.run.trial194.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T22:31:55.557 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T22:31:55.562 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T22:31:55.572 INFO:teuthology.orchestra.run.trial198.stdout:(32/35): thrift-0.15.0-4.el9.x86_64.rpm 24 MB/s | 1.6 MB 00:00 2026-02-22T22:31:55.703 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:31:55.704 INFO:teuthology.orchestra.run.trial198.stdout:(33/35): librados2-19.2.3-689.gf90f4eae.el9.x86 21 MB/s | 3.4 MB 00:00 2026-02-22T22:31:55.704 INFO:teuthology.orchestra.run.trial194.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:31:55.708 INFO:teuthology.orchestra.run.trial198.stdout:(34/35): librbd1-19.2.3-689.gf90f4eae.el9.x86_6 23 MB/s | 3.2 MB 00:00 2026-02-22T22:31:55.732 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:31:55.735 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T22:31:55.741 INFO:teuthology.orchestra.run.trial194.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T22:31:55.966 INFO:teuthology.orchestra.run.trial194.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T22:31:55.967 INFO:teuthology.orchestra.run.trial194.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:31:55.983 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:31:55.989 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T22:31:55.995 INFO:teuthology.orchestra.run.trial194.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T22:31:55.997 INFO:teuthology.orchestra.run.trial194.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T22:31:55.998 INFO:teuthology.orchestra.run.trial194.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:31:56.009 INFO:teuthology.orchestra.run.trial198.stdout:(35/35): nvme-cli-2.16-1.el9.x86_64.rpm 621 kB/s | 1.2 MB 00:01 2026-02-22T22:31:56.015 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:31:56.015 INFO:teuthology.orchestra.run.trial198.stdout:Total 18 MB/s | 63 MB 00:03 2026-02-22T22:31:56.017 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:31:56.110 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:31:56.128 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:31:56.128 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:31:56.135 INFO:teuthology.orchestra.run.trial194.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T22:31:56.137 INFO:teuthology.orchestra.run.trial194.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:31:56.381 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:31:56.382 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:31:56.430 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:31:56.430 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T22:31:56.430 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:31:56.466 INFO:teuthology.orchestra.run.trial194.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T22:31:56.468 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T22:31:56.671 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:31:56.680 INFO:teuthology.orchestra.run.trial198.stdout: Installing : lttng-ust-2.12.0-6.el9.x86_64 1/37 2026-02-22T22:31:56.682 INFO:teuthology.orchestra.run.trial198.stdout: Installing : librdkafka-1.6.1-102.el9.x86_64 2/37 2026-02-22T22:31:56.693 INFO:teuthology.orchestra.run.trial198.stdout: Installing : librabbitmq-0.11.0-7.el9.x86_64 3/37 2026-02-22T22:31:56.845 INFO:teuthology.orchestra.run.trial198.stdout: Installing : thrift-0.15.0-4.el9.x86_64 4/37 2026-02-22T22:31:56.849 INFO:teuthology.orchestra.run.trial198.stdout: Upgrading : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:31:56.879 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:31:56.922 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:31:56.923 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:31:56.936 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:31:56.940 INFO:teuthology.orchestra.run.trial198.stdout: Installing : liboath-2.6.12-1.el9.x86_64 8/37 2026-02-22T22:31:56.949 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 9/37 2026-02-22T22:31:56.971 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:31:56.971 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:31:56.983 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:31:57.015 INFO:teuthology.orchestra.run.trial198.stdout: Installing : re2-1:20211101-20.el9.x86_64 12/37 2026-02-22T22:31:57.019 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libarrow-9.0.0-15.el9.x86_64 13/37 2026-02-22T22:31:57.025 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libunwind-1.6.2-1.el9.x86_64 14/37 2026-02-22T22:31:57.027 INFO:teuthology.orchestra.run.trial198.stdout: Installing : gperftools-libs-2.9.1-3.el9.x86_64 15/37 2026-02-22T22:31:57.030 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libarrow-doc-9.0.0-15.el9.noarch 16/37 2026-02-22T22:31:57.036 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libpmemobj-1.12.1-1.el9.x86_64 17/37 2026-02-22T22:31:57.177 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:31:57.178 INFO:teuthology.orchestra.run.trial198.stdout: Upgrading : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:31:57.203 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 19/37 2026-02-22T22:31:57.205 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 20/37 2026-02-22T22:31:57.212 INFO:teuthology.orchestra.run.trial198.stdout: Installing : boost-program-options-1.75.0-13.el9.x86_64 21/37 2026-02-22T22:31:57.435 INFO:teuthology.orchestra.run.trial198.stdout: Installing : parquet-libs-9.0.0-15.el9.x86_64 22/37 2026-02-22T22:31:57.437 INFO:teuthology.orchestra.run.trial198.stdout: Installing : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:31:57.442 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:31:57.445 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:31:57.452 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 23/37 2026-02-22T22:31:57.456 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 24/37 2026-02-22T22:31:57.462 INFO:teuthology.orchestra.run.trial198.stdout: Installing : pciutils-3.7.0-7.el9.x86_64 25/37 2026-02-22T22:31:57.464 INFO:teuthology.orchestra.run.trial198.stdout: Installing : qatlib-25.08.0-2.el9.x86_64 26/37 2026-02-22T22:31:57.465 INFO:teuthology.orchestra.run.trial198.stdout: Installing : qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:31:57.482 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: qatlib-service-25.08.0-2.el9.x86_64 27/37 2026-02-22T22:31:57.596 INFO:teuthology.orchestra.run.trial198.stdout: Installing : qatzip-libs-1.3.1-1.el9.x86_64 28/37 2026-02-22T22:31:57.598 INFO:teuthology.orchestra.run.trial198.stdout: Installing : nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:31:57.727 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:31:57.730 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:31:57.781 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:31:57.781 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T22:31:57.782 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T22:31:57.782 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:31:57.784 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:31:57.871 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: nvme-cli-2.16-1.el9.x86_64 29/37 2026-02-22T22:31:57.872 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service → /usr/lib/systemd/system/nvmefc-boot-connections.service. 2026-02-22T22:31:57.872 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:31:57.909 INFO:teuthology.orchestra.run.trial198.stdout: Installing : mailcap-2.1.49-5.el9.noarch 30/37 2026-02-22T22:31:57.911 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 31/37 2026-02-22T22:31:58.892 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:31:58.895 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:31:59.175 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 32/37 2026-02-22T22:31:59.178 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:31:59.216 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 33/37 2026-02-22T22:31:59.216 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /usr/lib/systemd/system/ceph.target. 2026-02-22T22:31:59.216 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service → /usr/lib/systemd/system/ceph-crash.service. 2026-02-22T22:31:59.216 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:31:59.220 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:32:02.287 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:32:02.287 INFO:teuthology.orchestra.run.trial194.stdout:skipping the directory /sys 2026-02-22T22:32:02.288 INFO:teuthology.orchestra.run.trial194.stdout:skipping the directory /proc 2026-02-22T22:32:02.288 INFO:teuthology.orchestra.run.trial194.stdout:skipping the directory /mnt 2026-02-22T22:32:02.288 INFO:teuthology.orchestra.run.trial194.stdout:skipping the directory /var/tmp 2026-02-22T22:32:02.288 INFO:teuthology.orchestra.run.trial194.stdout:skipping the directory /home 2026-02-22T22:32:02.288 INFO:teuthology.orchestra.run.trial194.stdout:skipping the directory /root 2026-02-22T22:32:02.288 INFO:teuthology.orchestra.run.trial194.stdout:skipping the directory /tmp 2026-02-22T22:32:02.288 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:02.739 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:32:02.762 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial194.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:32:02.763 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:02.764 INFO:teuthology.orchestra.run.trial194.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:32:02.777 INFO:teuthology.orchestra.run.trial194.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial194.stdout:Creating group 'qat' with GID 992. 2026-02-22T22:32:02.833 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T22:32:03.116 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:32:03.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T22:32:03.118 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T22:32:03.119 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T22:32:03.119 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T22:32:03.119 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T22:32:03.119 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:32:03.119 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T22:32:03.119 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T22:32:03.175 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:32:03.175 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:03.175 INFO:teuthology.orchestra.run.trial194.stdout:Upgraded: 2026-02-22T22:32:03.175 INFO:teuthology.orchestra.run.trial194.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.176 INFO:teuthology.orchestra.run.trial194.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T22:32:03.177 INFO:teuthology.orchestra.run.trial194.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:03.178 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:03.217 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-test 2026-02-22T22:32:03.528 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:12 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:03.611 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:03.612 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:03.612 INFO:teuthology.orchestra.run.trial194.stdout: Package Arch Version Repository Size 2026-02-22T22:32:03.612 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:03.612 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:03.612 INFO:teuthology.orchestra.run.trial194.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout:Installing dependencies: 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout:Install 5 Packages 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:03.613 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 50 M 2026-02-22T22:32:03.614 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 212 M 2026-02-22T22:32:03.614 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout:skipping the directory /sys 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout:skipping the directory /proc 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout:skipping the directory /mnt 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout:skipping the directory /var/tmp 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout:skipping the directory /home 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout:skipping the directory /root 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout:skipping the directory /tmp 2026-02-22T22:32:03.758 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:03.782 INFO:teuthology.orchestra.run.trial194.stdout:(1/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 1.5 MB/s | 163 kB 00:00 2026-02-22T22:32:04.211 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:32:04.235 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 35/37 2026-02-22T22:32:04.235 INFO:teuthology.orchestra.run.trial198.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:04.235 INFO:teuthology.orchestra.run.trial198.stdout:Invalid unit name "ceph-radosgw@*.service" escaped as "ceph-radosgw@\x2a.service". 2026-02-22T22:32:04.235 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:32:04.236 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target → /usr/lib/systemd/system/ceph-radosgw.target. 2026-02-22T22:32:04.236 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:04.237 INFO:teuthology.orchestra.run.trial198.stdout: Cleanup : librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:32:04.250 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: librbd1-2:16.2.4-5.el9.x86_64 36/37 2026-02-22T22:32:04.250 INFO:teuthology.orchestra.run.trial198.stdout: Cleanup : librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:32:04.304 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: librados2-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:32:04.304 INFO:teuthology.orchestra.run.trial198.stdout:Creating group 'qat' with GID 992. 2026-02-22T22:32:04.304 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:04.325 INFO:teuthology.orchestra.run.trial194.stdout:(2/5): libxslt-1.1.34-12.el9.x86_64.rpm 359 kB/s | 233 kB 00:00 2026-02-22T22:32:04.408 INFO:teuthology.orchestra.run.trial194.stdout:(3/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 772 kB/s | 64 kB 00:00 2026-02-22T22:32:04.491 INFO:teuthology.orchestra.run.trial194.stdout:(4/5): socat-1.7.4.1-8.el9.x86_64.rpm 428 kB/s | 303 kB 00:00 2026-02-22T22:32:04.583 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 1/37 2026-02-22T22:32:04.583 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2/37 2026-02-22T22:32:04.583 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 3/37 2026-02-22T22:32:04.583 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 4/37 2026-02-22T22:32:04.583 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 5/37 2026-02-22T22:32:04.583 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 6/37 2026-02-22T22:32:04.583 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 7/37 2026-02-22T22:32:04.583 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x 8/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86 9/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 10/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 11/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 12/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 13/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : mailcap-2.1.49-5.el9.noarch 14/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : nvme-cli-2.16-1.el9.x86_64 15/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : pciutils-3.7.0-7.el9.x86_64 16/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : boost-program-options-1.75.0-13.el9.x86_64 17/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libnbd-1.20.3-4.el9.x86_64 18/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libpmemobj-1.12.1-1.el9.x86_64 19/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : librabbitmq-0.11.0-7.el9.x86_64 20/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : librdkafka-1.6.1-102.el9.x86_64 21/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : lttng-ust-2.12.0-6.el9.x86_64 22/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : qatlib-25.08.0-2.el9.x86_64 23/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : qatlib-service-25.08.0-2.el9.x86_64 24/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : qatzip-libs-1.3.1-1.el9.x86_64 25/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : gperftools-libs-2.9.1-3.el9.x86_64 26/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libarrow-9.0.0-15.el9.x86_64 27/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libarrow-doc-9.0.0-15.el9.noarch 28/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : liboath-2.6.12-1.el9.x86_64 29/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libunwind-1.6.2-1.el9.x86_64 30/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : parquet-libs-9.0.0-15.el9.x86_64 31/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : re2-1:20211101-20.el9.x86_64 32/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : thrift-0.15.0-4.el9.x86_64 33/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 34/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : librados2-2:16.2.4-5.el9.x86_64 35/37 2026-02-22T22:32:04.584 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 36/37 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : librbd1-2:16.2.4-5.el9.x86_64 37/37 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout:Upgraded: 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: boost-program-options-1.75.0-13.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: ceph-radosgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: ceph-selinux-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: gperftools-libs-2.9.1-3.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: libarrow-9.0.0-15.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: libarrow-doc-9.0.0-15.el9.noarch 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: libnbd-1.20.3-4.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: liboath-2.6.12-1.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: libpmemobj-1.12.1-1.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: librabbitmq-0.11.0-7.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: libradosstriper1-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: librdkafka-1.6.1-102.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: librgw2-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: libunwind-1.6.2-1.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: lttng-ust-2.12.0-6.el9.x86_64 2026-02-22T22:32:04.646 INFO:teuthology.orchestra.run.trial198.stdout: mailcap-2.1.49-5.el9.noarch 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: nvme-cli-2.16-1.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: parquet-libs-9.0.0-15.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: pciutils-3.7.0-7.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: python3-ceph-argparse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: python3-ceph-common-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: qatlib-25.08.0-2.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: qatlib-service-25.08.0-2.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: qatzip-libs-1.3.1-1.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: re2-1:20211101-20.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: thrift-0.15.0-4.el9.x86_64 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:04.647 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:04.688 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-test 2026-02-22T22:32:04.702 INFO:teuthology.orchestra.run.trial194.stdout:(5/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 48 MB/s | 50 MB 00:01 2026-02-22T22:32:04.704 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:04.705 INFO:teuthology.orchestra.run.trial194.stdout:Total 46 MB/s | 50 MB 00:01 2026-02-22T22:32:04.732 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:04.741 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:04.742 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:04.927 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:04.927 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:05.002 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:14 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout: Package Arch Version Repository Size 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout: ceph-test x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 50 M 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:Installing dependencies: 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout: libcephsqlite x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 163 k 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout: libxslt x86_64 1.1.34-12.el9 appstream 233 k 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout: socat x86_64 1.7.4.1-8.el9 appstream 303 k 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout: xmlstarlet x86_64 1.6.1-20.el9 appstream 64 k 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:Install 5 Packages 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 50 M 2026-02-22T22:32:05.085 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 212 M 2026-02-22T22:32:05.086 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:05.139 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:05.144 INFO:teuthology.orchestra.run.trial194.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T22:32:05.149 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T22:32:05.157 INFO:teuthology.orchestra.run.trial194.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T22:32:05.157 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:32:05.264 INFO:teuthology.orchestra.run.trial198.stdout:(1/5): libcephsqlite-19.2.3-689.gf90f4eae.el9.x 1.8 MB/s | 163 kB 00:00 2026-02-22T22:32:05.949 INFO:teuthology.orchestra.run.trial198.stdout:(2/5): libxslt-1.1.34-12.el9.x86_64.rpm 302 kB/s | 233 kB 00:00 2026-02-22T22:32:06.050 INFO:teuthology.orchestra.run.trial198.stdout:(3/5): socat-1.7.4.1-8.el9.x86_64.rpm 386 kB/s | 303 kB 00:00 2026-02-22T22:32:06.185 INFO:teuthology.orchestra.run.trial198.stdout:(4/5): ceph-test-19.2.3-689.gf90f4eae.el9.x86_6 49 MB/s | 50 MB 00:01 2026-02-22T22:32:06.579 INFO:teuthology.orchestra.run.trial198.stdout:(5/5): xmlstarlet-1.6.1-20.el9.x86_64.rpm 101 kB/s | 64 kB 00:00 2026-02-22T22:32:06.581 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:06.582 INFO:teuthology.orchestra.run.trial198.stdout:Total 34 MB/s | 50 MB 00:01 2026-02-22T22:32:06.611 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:06.620 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:06.620 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:06.806 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:06.806 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:07.018 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:07.024 INFO:teuthology.orchestra.run.trial198.stdout: Installing : socat-1.7.4.1-8.el9.x86_64 1/5 2026-02-22T22:32:07.028 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libxslt-1.1.34-12.el9.x86_64 2/5 2026-02-22T22:32:07.036 INFO:teuthology.orchestra.run.trial198.stdout: Installing : xmlstarlet-1.6.1-20.el9.x86_64 3/5 2026-02-22T22:32:07.036 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:32:07.407 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:32:07.424 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:32:07.659 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:32:07.660 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T22:32:07.660 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T22:32:07.660 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T22:32:07.660 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T22:32:07.713 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T22:32:07.713 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:07.713 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:07.713 INFO:teuthology.orchestra.run.trial194.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:07.714 INFO:teuthology.orchestra.run.trial194.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:07.714 INFO:teuthology.orchestra.run.trial194.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T22:32:07.714 INFO:teuthology.orchestra.run.trial194.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T22:32:07.714 INFO:teuthology.orchestra.run.trial194.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T22:32:07.714 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:07.714 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:07.745 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph 2026-02-22T22:32:08.047 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:17 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:08.133 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:08.136 INFO:teuthology.orchestra.run.trial194.stdout:====================================================================================== 2026-02-22T22:32:08.136 INFO:teuthology.orchestra.run.trial194.stdout: Package Arch Version Repository Size 2026-02-22T22:32:08.136 INFO:teuthology.orchestra.run.trial194.stdout:====================================================================================== 2026-02-22T22:32:08.136 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout:Installing dependencies: 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T22:32:08.137 INFO:teuthology.orchestra.run.trial194.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T22:32:08.138 INFO:teuthology.orchestra.run.trial194.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T22:32:08.139 INFO:teuthology.orchestra.run.trial194.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T22:32:08.140 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T22:32:08.141 INFO:teuthology.orchestra.run.trial194.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T22:32:08.142 INFO:teuthology.orchestra.run.trial194.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T22:32:08.143 INFO:teuthology.orchestra.run.trial194.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout:Installing weak dependencies: 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T22:32:08.144 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:08.145 INFO:teuthology.orchestra.run.trial194.stdout:====================================================================================== 2026-02-22T22:32:08.146 INFO:teuthology.orchestra.run.trial194.stdout:Install 99 Packages 2026-02-22T22:32:08.146 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:08.147 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 95 M 2026-02-22T22:32:08.147 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 520 M 2026-02-22T22:32:08.147 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:08.691 INFO:teuthology.orchestra.run.trial194.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 112 kB/s | 6.5 kB 00:00 2026-02-22T22:32:08.767 INFO:teuthology.orchestra.run.trial194.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 8.0 MB/s | 1.1 MB 00:00 2026-02-22T22:32:08.790 INFO:teuthology.orchestra.run.trial194.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 15 MB/s | 2.4 MB 00:00 2026-02-22T22:32:08.820 INFO:teuthology.orchestra.run.trial194.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 1.0 MB/s | 31 kB 00:00 2026-02-22T22:32:08.855 INFO:teuthology.orchestra.run.trial194.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 4.2 MB/s | 150 kB 00:00 2026-02-22T22:32:09.009 INFO:teuthology.orchestra.run.trial194.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 15 MB/s | 4.7 MB 00:00 2026-02-22T22:32:09.028 INFO:teuthology.orchestra.run.trial194.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 22 MB/s | 3.8 MB 00:00 2026-02-22T22:32:09.067 INFO:teuthology.orchestra.run.trial194.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 567 kB/s | 22 kB 00:00 2026-02-22T22:32:09.113 INFO:teuthology.orchestra.run.trial194.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 5.4 MB/s | 253 kB 00:00 2026-02-22T22:32:09.156 INFO:teuthology.orchestra.run.trial194.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 1.1 MB/s | 49 kB 00:00 2026-02-22T22:32:09.202 INFO:teuthology.orchestra.run.trial194.stdout:(11/99): ceph-prometheus-alerts-19.2.3-689.gf90 368 kB/s | 17 kB 00:00 2026-02-22T22:32:09.256 INFO:teuthology.orchestra.run.trial194.stdout:(12/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 5.4 MB/s | 299 kB 00:00 2026-02-22T22:32:09.292 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 4/5 2026-02-22T22:32:09.310 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:32:09.321 INFO:teuthology.orchestra.run.trial194.stdout:(13/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 12 MB/s | 769 kB 00:00 2026-02-22T22:32:09.387 INFO:teuthology.orchestra.run.trial194.stdout:(14/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 28 MB/s | 17 MB 00:00 2026-02-22T22:32:09.397 INFO:teuthology.orchestra.run.trial194.stdout:(15/99): ceph-mgr-diskprediction-local-19.2.3-6 19 MB/s | 7.4 MB 00:00 2026-02-22T22:32:09.534 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 5/5 2026-02-22T22:32:09.535 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 1/5 2026-02-22T22:32:09.535 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2/5 2026-02-22T22:32:09.535 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libxslt-1.1.34-12.el9.x86_64 3/5 2026-02-22T22:32:09.535 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : socat-1.7.4.1-8.el9.x86_64 4/5 2026-02-22T22:32:09.587 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : xmlstarlet-1.6.1-20.el9.x86_64 5/5 2026-02-22T22:32:09.588 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:09.588 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:09.588 INFO:teuthology.orchestra.run.trial198.stdout: ceph-test-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:09.588 INFO:teuthology.orchestra.run.trial198.stdout: libcephsqlite-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:09.588 INFO:teuthology.orchestra.run.trial198.stdout: libxslt-1.1.34-12.el9.x86_64 2026-02-22T22:32:09.588 INFO:teuthology.orchestra.run.trial198.stdout: socat-1.7.4.1-8.el9.x86_64 2026-02-22T22:32:09.588 INFO:teuthology.orchestra.run.trial198.stdout: xmlstarlet-1.6.1-20.el9.x86_64 2026-02-22T22:32:09.588 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:09.589 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:09.623 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph 2026-02-22T22:32:09.793 INFO:teuthology.orchestra.run.trial194.stdout:(16/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 100 kB/s | 40 kB 00:00 2026-02-22T22:32:09.882 INFO:teuthology.orchestra.run.trial194.stdout:(17/99): libconfig-1.7.2-9.el9.x86_64.rpm 148 kB/s | 72 kB 00:00 2026-02-22T22:32:09.929 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:18 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:10.016 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:10.019 INFO:teuthology.orchestra.run.trial198.stdout:====================================================================================== 2026-02-22T22:32:10.019 INFO:teuthology.orchestra.run.trial198.stdout: Package Arch Version Repository Size 2026-02-22T22:32:10.019 INFO:teuthology.orchestra.run.trial198.stdout:====================================================================================== 2026-02-22T22:32:10.019 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: ceph x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 6.5 k 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout:Installing dependencies: 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: abseil-cpp x86_64 20211102.0-4.el9 epel 551 k 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: ceph-grafana-dashboards noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 31 k 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mds x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 2.4 M 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.1 M 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-modules-core noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 253 k 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mon x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 4.7 M 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: ceph-osd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 17 M 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: ceph-prometheus-alerts noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 17 k 2026-02-22T22:32:10.020 INFO:teuthology.orchestra.run.trial198.stdout: cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 769 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: cryptsetup x86_64 2.8.1-3.el9 baseos 351 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: flexiblas x86_64 3.0.4-9.el9 appstream 30 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: flexiblas-netlib x86_64 3.0.4-9.el9 appstream 3.0 M 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: flexiblas-openblas-openmp x86_64 3.0.4-9.el9 appstream 15 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: grpc-data noarch 1.46.7-10.el9 epel 19 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: ledmon-libs x86_64 1.1.0-3.el9 baseos 40 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: libconfig x86_64 1.7.2-9.el9 baseos 72 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: libgfortran x86_64 11.5.0-14.el9 baseos 794 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: libquadmath x86_64 11.5.0-14.el9 baseos 184 k 2026-02-22T22:32:10.021 INFO:teuthology.orchestra.run.trial198.stdout: libstoragemgmt x86_64 1.10.1-1.el9 appstream 246 k 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: lua x86_64 5.4.4-4.el9 appstream 188 k 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: lua-devel x86_64 5.4.4-4.el9 crb 22 k 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: luarocks noarch 3.9.2-5.el9 epel 151 k 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: openblas x86_64 0.3.29-1.el9 appstream 42 k 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: openblas-openmp x86_64 0.3.29-1.el9 appstream 5.3 M 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: protobuf x86_64 3.14.0-17.el9 appstream 1.0 M 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: protobuf-compiler x86_64 3.14.0-17.el9 crb 862 k 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: python3-asyncssh noarch 2.13.2-5.el9 epel 548 k 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: python3-autocommand noarch 2.2.2-8.el9 epel 29 k 2026-02-22T22:32:10.022 INFO:teuthology.orchestra.run.trial198.stdout: python3-babel noarch 2.9.1-2.el9 appstream 6.0 M 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-backports-tarfile noarch 1.2.0-1.el9 epel 60 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-bcrypt x86_64 3.2.2-1.el9 epel 43 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-cachetools noarch 4.2.4-1.el9 epel 32 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-certifi noarch 2023.05.07-4.el9 epel 14 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-cffi x86_64 1.14.5-5.el9 baseos 253 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-cheroot noarch 10.0.1-4.el9 epel 173 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-cherrypy noarch 18.6.1-2.el9 epel 358 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-cryptography x86_64 36.0.1-5.el9 baseos 1.2 M 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-devel x86_64 3.9.25-3.el9 appstream 244 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-google-auth noarch 1:2.45.0-1.el9 epel 254 k 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-grpcio x86_64 1.46.7-10.el9 epel 2.0 M 2026-02-22T22:32:10.023 INFO:teuthology.orchestra.run.trial198.stdout: python3-grpcio-tools x86_64 1.46.7-10.el9 epel 144 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-isodate noarch 0.6.1-3.el9 epel 56 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco noarch 8.2.1-3.el9 epel 11 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-classes noarch 3.2.1-5.el9 epel 18 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-collections noarch 3.0.0-8.el9 epel 23 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-context noarch 6.0.1-3.el9 epel 20 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-functools noarch 3.5.0-2.el9 epel 19 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-text noarch 4.0.0-2.el9 epel 26 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-jinja2 noarch 2.11.3-8.el9 appstream 249 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-kubernetes noarch 1:26.1.0-3.el9 epel 1.0 M 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-libstoragemgmt x86_64 1.10.1-1.el9 appstream 177 k 2026-02-22T22:32:10.024 INFO:teuthology.orchestra.run.trial198.stdout: python3-logutils noarch 0.3.5-21.el9 epel 46 k 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-lxml x86_64 4.6.5-3.el9 appstream 1.2 M 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-mako noarch 1.1.4-6.el9 appstream 172 k 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-markupsafe x86_64 1.1.1-12.el9 appstream 35 k 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-more-itertools noarch 8.12.0-2.el9 epel 79 k 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-msgpack x86_64 1.0.3-2.el9 epel 86 k 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-natsort noarch 7.1.1-5.el9 epel 58 k 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-numpy x86_64 1:1.23.5-2.el9 appstream 6.1 M 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-numpy-f2py x86_64 1:1.23.5-2.el9 appstream 442 k 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-packaging noarch 20.9-5.el9 appstream 77 k 2026-02-22T22:32:10.025 INFO:teuthology.orchestra.run.trial198.stdout: python3-pecan noarch 1.4.2-3.el9 epel 272 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-ply noarch 3.11-14.el9 baseos 106 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-portend noarch 3.1.0-2.el9 epel 16 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-protobuf noarch 3.14.0-17.el9 appstream 267 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-pyOpenSSL noarch 21.0.0-1.el9 epel 90 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-pyasn1 noarch 0.4.8-6.el9 appstream 159 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-pyasn1-modules noarch 0.4.8-6.el9 appstream 279 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-pycparser noarch 2.20-6.el9 baseos 135 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-pyparsing noarch 2.4.7-9.el9 baseos 150 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-repoze-lru noarch 0.7-16.el9 epel 31 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-requests noarch 2.25.1-10.el9 baseos 126 k 2026-02-22T22:32:10.026 INFO:teuthology.orchestra.run.trial198.stdout: python3-requests-oauthlib noarch 1.3.0-12.el9 appstream 54 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-routes noarch 2.5.1-5.el9 epel 188 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-rsa noarch 4.9-2.el9 epel 59 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-scipy x86_64 1.9.3-2.el9 appstream 19 M 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-tempora noarch 5.0.0-2.el9 epel 36 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-toml noarch 0.10.2-6.el9 appstream 42 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-typing-extensions noarch 4.15.0-1.el9 epel 86 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-urllib3 noarch 1.26.5-7.el9 baseos 218 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-webob noarch 1.8.8-2.el9 epel 230 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-websocket-client noarch 1.2.3-2.el9 epel 90 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-werkzeug noarch 2.0.3-3.el9.1 epel 427 k 2026-02-22T22:32:10.027 INFO:teuthology.orchestra.run.trial198.stdout: python3-xmlsec x86_64 1.3.13-1.el9 epel 48 k 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: python3-xmltodict noarch 0.12.0-15.el9 epel 22 k 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: python3-zc-lockfile noarch 2.0-10.el9 epel 20 k 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: unzip x86_64 6.0-59.el9 baseos 182 k 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: xmlsec1 x86_64 1.2.29-13.el9 appstream 189 k 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: xmlsec1-openssl x86_64 1.2.29-13.el9 appstream 90 k 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: zip x86_64 3.0-35.el9 baseos 266 k 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout:Installing weak dependencies: 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-cephadm noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 150 k 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-dashboard noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 3.8 M 2026-02-22T22:32:10.028 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-diskprediction-local noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 7.4 M 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-k8sevents noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 22 k 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-rook noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 49 k 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout: ceph-volume noarch 2:19.2.3-689.gf90f4eae.el9 ceph-noarch 299 k 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout: python3-influxdb noarch 5.3.1-1.el9 epel 139 k 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout: python3-saml noarch 1.16.0-1.el9 epel 125 k 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout:====================================================================================== 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout:Install 99 Packages 2026-02-22T22:32:10.029 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:10.030 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 95 M 2026-02-22T22:32:10.030 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 520 M 2026-02-22T22:32:10.031 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:10.125 INFO:teuthology.orchestra.run.trial194.stdout:(18/99): libquadmath-11.5.0-14.el9.x86_64.rpm 762 kB/s | 184 kB 00:00 2026-02-22T22:32:10.201 INFO:teuthology.orchestra.run.trial194.stdout:(19/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 399 kB/s | 351 kB 00:00 2026-02-22T22:32:10.396 INFO:teuthology.orchestra.run.trial198.stdout:(1/99): ceph-19.2.3-689.gf90f4eae.el9.x86_64.rp 127 kB/s | 6.5 kB 00:00 2026-02-22T22:32:10.447 INFO:teuthology.orchestra.run.trial194.stdout:(20/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 786 kB/s | 253 kB 00:00 2026-02-22T22:32:10.488 INFO:teuthology.orchestra.run.trial198.stdout:(2/99): ceph-mgr-19.2.3-689.gf90f4eae.el9.x86_6 7.5 MB/s | 1.1 MB 00:00 2026-02-22T22:32:10.514 INFO:teuthology.orchestra.run.trial198.stdout:(3/99): ceph-mds-19.2.3-689.gf90f4eae.el9.x86_6 14 MB/s | 2.4 MB 00:00 2026-02-22T22:32:10.548 INFO:teuthology.orchestra.run.trial198.stdout:(4/99): ceph-grafana-dashboards-19.2.3-689.gf90 927 kB/s | 31 kB 00:00 2026-02-22T22:32:10.586 INFO:teuthology.orchestra.run.trial198.stdout:(5/99): ceph-mgr-cephadm-19.2.3-689.gf90f4eae.e 3.9 MB/s | 150 kB 00:00 2026-02-22T22:32:10.610 INFO:teuthology.orchestra.run.trial194.stdout:(21/99): python3-ply-3.11-14.el9.noarch.rpm 652 kB/s | 106 kB 00:00 2026-02-22T22:32:10.622 INFO:teuthology.orchestra.run.trial198.stdout:(6/99): ceph-mon-19.2.3-689.gf90f4eae.el9.x86_6 21 MB/s | 4.7 MB 00:00 2026-02-22T22:32:10.768 INFO:teuthology.orchestra.run.trial198.stdout:(7/99): ceph-mgr-dashboard-19.2.3-689.gf90f4eae 21 MB/s | 3.8 MB 00:00 2026-02-22T22:32:10.774 INFO:teuthology.orchestra.run.trial194.stdout:(22/99): python3-pycparser-2.20-6.el9.noarch.rp 827 kB/s | 135 kB 00:00 2026-02-22T22:32:10.812 INFO:teuthology.orchestra.run.trial198.stdout:(8/99): ceph-mgr-k8sevents-19.2.3-689.gf90f4eae 504 kB/s | 22 kB 00:00 2026-02-22T22:32:10.862 INFO:teuthology.orchestra.run.trial198.stdout:(9/99): ceph-mgr-modules-core-19.2.3-689.gf90f4 4.9 MB/s | 253 kB 00:00 2026-02-22T22:32:10.910 INFO:teuthology.orchestra.run.trial198.stdout:(10/99): ceph-mgr-rook-19.2.3-689.gf90f4eae.el9 1.0 MB/s | 49 kB 00:00 2026-02-22T22:32:10.949 INFO:teuthology.orchestra.run.trial198.stdout:(11/99): ceph-mgr-diskprediction-local-19.2.3-6 23 MB/s | 7.4 MB 00:00 2026-02-22T22:32:10.950 INFO:teuthology.orchestra.run.trial198.stdout:(12/99): ceph-prometheus-alerts-19.2.3-689.gf90 425 kB/s | 17 kB 00:00 2026-02-22T22:32:10.990 INFO:teuthology.orchestra.run.trial198.stdout:(13/99): cephadm-19.2.3-689.gf90f4eae.el9.noarc 19 MB/s | 769 kB 00:00 2026-02-22T22:32:10.995 INFO:teuthology.orchestra.run.trial198.stdout:(14/99): ceph-volume-19.2.3-689.gf90f4eae.el9.n 6.2 MB/s | 299 kB 00:00 2026-02-22T22:32:11.016 INFO:teuthology.orchestra.run.trial194.stdout:(23/99): python3-pyparsing-2.4.7-9.el9.noarch.r 621 kB/s | 150 kB 00:00 2026-02-22T22:32:11.038 INFO:teuthology.orchestra.run.trial198.stdout:(15/99): ledmon-libs-1.1.0-3.el9.x86_64.rpm 948 kB/s | 40 kB 00:00 2026-02-22T22:32:11.058 INFO:teuthology.orchestra.run.trial198.stdout:(16/99): libconfig-1.7.2-9.el9.x86_64.rpm 3.5 MB/s | 72 kB 00:00 2026-02-22T22:32:11.071 INFO:teuthology.orchestra.run.trial198.stdout:(17/99): cryptsetup-2.8.1-3.el9.x86_64.rpm 4.2 MB/s | 351 kB 00:00 2026-02-22T22:32:11.083 INFO:teuthology.orchestra.run.trial198.stdout:(18/99): libquadmath-11.5.0-14.el9.x86_64.rpm 15 MB/s | 184 kB 00:00 2026-02-22T22:32:11.107 INFO:teuthology.orchestra.run.trial198.stdout:(19/99): ceph-osd-19.2.3-689.gf90f4eae.el9.x86_ 28 MB/s | 17 MB 00:00 2026-02-22T22:32:11.108 INFO:teuthology.orchestra.run.trial198.stdout:(20/99): python3-cffi-1.14.5-5.el9.x86_64.rpm 10 MB/s | 253 kB 00:00 2026-02-22T22:32:11.109 INFO:teuthology.orchestra.run.trial198.stdout:(21/99): libgfortran-11.5.0-14.el9.x86_64.rpm 15 MB/s | 794 kB 00:00 2026-02-22T22:32:11.120 INFO:teuthology.orchestra.run.trial198.stdout:(22/99): python3-ply-3.11-14.el9.noarch.rpm 8.9 MB/s | 106 kB 00:00 2026-02-22T22:32:11.120 INFO:teuthology.orchestra.run.trial198.stdout:(23/99): python3-pycparser-2.20-6.el9.noarch.rp 12 MB/s | 135 kB 00:00 2026-02-22T22:32:11.130 INFO:teuthology.orchestra.run.trial198.stdout:(24/99): python3-requests-2.25.1-10.el9.noarch. 13 MB/s | 126 kB 00:00 2026-02-22T22:32:11.132 INFO:teuthology.orchestra.run.trial198.stdout:(25/99): python3-pyparsing-2.4.7-9.el9.noarch.r 12 MB/s | 150 kB 00:00 2026-02-22T22:32:11.142 INFO:teuthology.orchestra.run.trial198.stdout:(26/99): python3-urllib3-1.26.5-7.el9.noarch.rp 18 MB/s | 218 kB 00:00 2026-02-22T22:32:11.145 INFO:teuthology.orchestra.run.trial198.stdout:(27/99): unzip-6.0-59.el9.x86_64.rpm 14 MB/s | 182 kB 00:00 2026-02-22T22:32:11.154 INFO:teuthology.orchestra.run.trial198.stdout:(28/99): zip-3.0-35.el9.x86_64.rpm 22 MB/s | 266 kB 00:00 2026-02-22T22:32:11.180 INFO:teuthology.orchestra.run.trial194.stdout:(24/99): python3-requests-2.25.1-10.el9.noarch. 772 kB/s | 126 kB 00:00 2026-02-22T22:32:11.187 INFO:teuthology.orchestra.run.trial198.stdout:(29/99): python3-cryptography-36.0.1-5.el9.x86_ 16 MB/s | 1.2 MB 00:00 2026-02-22T22:32:11.502 INFO:teuthology.orchestra.run.trial194.stdout:(25/99): python3-urllib3-1.26.5-7.el9.noarch.rp 677 kB/s | 218 kB 00:00 2026-02-22T22:32:11.702 INFO:teuthology.orchestra.run.trial194.stdout:(26/99): python3-cryptography-36.0.1-5.el9.x86_ 850 kB/s | 1.2 MB 00:01 2026-02-22T22:32:11.703 INFO:teuthology.orchestra.run.trial198.stdout:(30/99): flexiblas-3.0.4-9.el9.x86_64.rpm 53 kB/s | 30 kB 00:00 2026-02-22T22:32:11.735 INFO:teuthology.orchestra.run.trial194.stdout:(27/99): libgfortran-11.5.0-14.el9.x86_64.rpm 409 kB/s | 794 kB 00:01 2026-02-22T22:32:11.739 INFO:teuthology.orchestra.run.trial198.stdout:(31/99): flexiblas-openblas-openmp-3.0.4-9.el9. 27 kB/s | 15 kB 00:00 2026-02-22T22:32:11.745 INFO:teuthology.orchestra.run.trial194.stdout:(28/99): unzip-6.0-59.el9.x86_64.rpm 750 kB/s | 182 kB 00:00 2026-02-22T22:32:11.970 INFO:teuthology.orchestra.run.trial194.stdout:(29/99): flexiblas-3.0.4-9.el9.x86_64.rpm 126 kB/s | 30 kB 00:00 2026-02-22T22:32:12.022 INFO:teuthology.orchestra.run.trial194.stdout:(30/99): zip-3.0-35.el9.x86_64.rpm 831 kB/s | 266 kB 00:00 2026-02-22T22:32:12.096 INFO:teuthology.orchestra.run.trial198.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 627 kB/s | 246 kB 00:00 2026-02-22T22:32:12.128 INFO:teuthology.orchestra.run.trial198.stdout:(33/99): lua-5.4.4-4.el9.x86_64.rpm 483 kB/s | 188 kB 00:00 2026-02-22T22:32:12.129 INFO:teuthology.orchestra.run.trial194.stdout:(31/99): flexiblas-openblas-openmp-3.0.4-9.el9. 94 kB/s | 15 kB 00:00 2026-02-22T22:32:12.256 INFO:teuthology.orchestra.run.trial198.stdout:(34/99): openblas-0.3.29-1.el9.x86_64.rpm 264 kB/s | 42 kB 00:00 2026-02-22T22:32:12.413 INFO:teuthology.orchestra.run.trial194.stdout:(32/99): libstoragemgmt-1.10.1-1.el9.x86_64.rpm 628 kB/s | 246 kB 00:00 2026-02-22T22:32:12.445 INFO:teuthology.orchestra.run.trial194.stdout:(33/99): lua-5.4.4-4.el9.x86_64.rpm 597 kB/s | 188 kB 00:00 2026-02-22T22:32:12.572 INFO:teuthology.orchestra.run.trial194.stdout:(34/99): openblas-0.3.29-1.el9.x86_64.rpm 266 kB/s | 42 kB 00:00 2026-02-22T22:32:13.507 INFO:teuthology.orchestra.run.trial198.stdout:(35/99): protobuf-3.14.0-17.el9.x86_64.rpm 822 kB/s | 1.0 MB 00:01 2026-02-22T22:32:13.832 INFO:teuthology.orchestra.run.trial194.stdout:(35/99): protobuf-3.14.0-17.el9.x86_64.rpm 816 kB/s | 1.0 MB 00:01 2026-02-22T22:32:15.085 INFO:teuthology.orchestra.run.trial198.stdout:(36/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 779 kB/s | 3.0 MB 00:03 2026-02-22T22:32:15.392 INFO:teuthology.orchestra.run.trial194.stdout:(36/99): flexiblas-netlib-3.0.4-9.el9.x86_64.rp 839 kB/s | 3.0 MB 00:03 2026-02-22T22:32:15.481 INFO:teuthology.orchestra.run.trial198.stdout:(37/99): python3-devel-3.9.25-3.el9.x86_64.rpm 618 kB/s | 244 kB 00:00 2026-02-22T22:32:15.789 INFO:teuthology.orchestra.run.trial194.stdout:(37/99): python3-devel-3.9.25-3.el9.x86_64.rpm 616 kB/s | 244 kB 00:00 2026-02-22T22:32:15.876 INFO:teuthology.orchestra.run.trial198.stdout:(38/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 629 kB/s | 249 kB 00:00 2026-02-22T22:32:16.186 INFO:teuthology.orchestra.run.trial194.stdout:(38/99): python3-jinja2-2.11.3-8.el9.noarch.rpm 626 kB/s | 249 kB 00:00 2026-02-22T22:32:16.193 INFO:teuthology.orchestra.run.trial198.stdout:(39/99): python3-libstoragemgmt-1.10.1-1.el9.x8 558 kB/s | 177 kB 00:00 2026-02-22T22:32:16.504 INFO:teuthology.orchestra.run.trial194.stdout:(39/99): python3-libstoragemgmt-1.10.1-1.el9.x8 556 kB/s | 177 kB 00:00 2026-02-22T22:32:17.687 INFO:teuthology.orchestra.run.trial198.stdout:(40/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 838 kB/s | 1.2 MB 00:01 2026-02-22T22:32:18.004 INFO:teuthology.orchestra.run.trial198.stdout:(41/99): python3-mako-1.1.4-6.el9.noarch.rpm 544 kB/s | 172 kB 00:00 2026-02-22T22:32:18.011 INFO:teuthology.orchestra.run.trial194.stdout:(40/99): python3-lxml-4.6.5-3.el9.x86_64.rpm 831 kB/s | 1.2 MB 00:01 2026-02-22T22:32:18.164 INFO:teuthology.orchestra.run.trial198.stdout:(42/99): python3-markupsafe-1.1.1-12.el9.x86_64 217 kB/s | 35 kB 00:00 2026-02-22T22:32:18.185 INFO:teuthology.orchestra.run.trial198.stdout:(43/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 893 kB/s | 5.3 MB 00:06 2026-02-22T22:32:18.328 INFO:teuthology.orchestra.run.trial194.stdout:(41/99): python3-mako-1.1.4-6.el9.noarch.rpm 543 kB/s | 172 kB 00:00 2026-02-22T22:32:18.347 INFO:teuthology.orchestra.run.trial194.stdout:(42/99): openblas-openmp-0.3.29-1.el9.x86_64.rp 917 kB/s | 5.3 MB 00:05 2026-02-22T22:32:18.487 INFO:teuthology.orchestra.run.trial194.stdout:(43/99): python3-markupsafe-1.1.1-12.el9.x86_64 219 kB/s | 35 kB 00:00 2026-02-22T22:32:18.888 INFO:teuthology.orchestra.run.trial198.stdout:(44/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 629 kB/s | 442 kB 00:00 2026-02-22T22:32:19.056 INFO:teuthology.orchestra.run.trial198.stdout:(45/99): python3-babel-2.9.1-2.el9.noarch.rpm 1.1 MB/s | 6.0 MB 00:05 2026-02-22T22:32:19.121 INFO:teuthology.orchestra.run.trial194.stdout:(44/99): python3-numpy-f2py-1.23.5-2.el9.x86_64 697 kB/s | 442 kB 00:00 2026-02-22T22:32:19.125 INFO:teuthology.orchestra.run.trial198.stdout:(46/99): python3-packaging-20.9-5.el9.noarch.rp 326 kB/s | 77 kB 00:00 2026-02-22T22:32:19.217 INFO:teuthology.orchestra.run.trial198.stdout:(47/99): python3-protobuf-3.14.0-17.el9.noarch. 1.6 MB/s | 267 kB 00:00 2026-02-22T22:32:19.360 INFO:teuthology.orchestra.run.trial194.stdout:(45/99): python3-packaging-20.9-5.el9.noarch.rp 324 kB/s | 77 kB 00:00 2026-02-22T22:32:19.377 INFO:teuthology.orchestra.run.trial198.stdout:(48/99): python3-pyasn1-modules-0.4.8-6.el9.noa 1.7 MB/s | 279 kB 00:00 2026-02-22T22:32:19.439 INFO:teuthology.orchestra.run.trial198.stdout:(49/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 506 kB/s | 159 kB 00:00 2026-02-22T22:32:19.537 INFO:teuthology.orchestra.run.trial198.stdout:(50/99): python3-requests-oauthlib-1.3.0-12.el9 337 kB/s | 54 kB 00:00 2026-02-22T22:32:19.697 INFO:teuthology.orchestra.run.trial198.stdout:(51/99): python3-toml-0.10.2-6.el9.noarch.rpm 260 kB/s | 42 kB 00:00 2026-02-22T22:32:19.757 INFO:teuthology.orchestra.run.trial194.stdout:(46/99): python3-protobuf-3.14.0-17.el9.noarch. 673 kB/s | 267 kB 00:00 2026-02-22T22:32:19.857 INFO:teuthology.orchestra.run.trial198.stdout:(52/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 1.1 MB/s | 189 kB 00:00 2026-02-22T22:32:20.017 INFO:teuthology.orchestra.run.trial198.stdout:(53/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 566 kB/s | 90 kB 00:00 2026-02-22T22:32:20.076 INFO:teuthology.orchestra.run.trial194.stdout:(47/99): python3-pyasn1-0.4.8-6.el9.noarch.rpm 499 kB/s | 159 kB 00:00 2026-02-22T22:32:20.221 INFO:teuthology.orchestra.run.trial198.stdout:(54/99): lua-devel-5.4.4-4.el9.x86_64.rpm 109 kB/s | 22 kB 00:00 2026-02-22T22:32:20.473 INFO:teuthology.orchestra.run.trial194.stdout:(48/99): python3-pyasn1-modules-0.4.8-6.el9.noa 702 kB/s | 279 kB 00:00 2026-02-22T22:32:20.604 INFO:teuthology.orchestra.run.trial194.stdout:(49/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 2.7 MB/s | 6.1 MB 00:02 2026-02-22T22:32:20.664 INFO:teuthology.orchestra.run.trial194.stdout:(50/99): python3-requests-oauthlib-1.3.0-12.el9 279 kB/s | 54 kB 00:00 2026-02-22T22:32:20.760 INFO:teuthology.orchestra.run.trial194.stdout:(51/99): python3-babel-2.9.1-2.el9.noarch.rpm 881 kB/s | 6.0 MB 00:06 2026-02-22T22:32:20.823 INFO:teuthology.orchestra.run.trial194.stdout:(52/99): python3-toml-0.10.2-6.el9.noarch.rpm 262 kB/s | 42 kB 00:00 2026-02-22T22:32:21.060 INFO:teuthology.orchestra.run.trial194.stdout:(53/99): xmlsec1-openssl-1.2.29-13.el9.x86_64.r 381 kB/s | 90 kB 00:00 2026-02-22T22:32:21.076 INFO:teuthology.orchestra.run.trial194.stdout:(54/99): xmlsec1-1.2.29-13.el9.x86_64.rpm 597 kB/s | 189 kB 00:00 2026-02-22T22:32:21.085 INFO:teuthology.orchestra.run.trial198.stdout:(55/99): protobuf-compiler-3.14.0-17.el9.x86_64 998 kB/s | 862 kB 00:00 2026-02-22T22:32:21.147 INFO:teuthology.orchestra.run.trial194.stdout:(55/99): lua-devel-5.4.4-4.el9.x86_64.rpm 258 kB/s | 22 kB 00:00 2026-02-22T22:32:21.395 INFO:teuthology.orchestra.run.trial194.stdout:(56/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 2.2 MB/s | 551 kB 00:00 2026-02-22T22:32:21.414 INFO:teuthology.orchestra.run.trial198.stdout:(56/99): abseil-cpp-20211102.0-4.el9.x86_64.rpm 1.6 MB/s | 551 kB 00:00 2026-02-22T22:32:21.421 INFO:teuthology.orchestra.run.trial194.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 741 kB/s | 19 kB 00:00 2026-02-22T22:32:21.449 INFO:teuthology.orchestra.run.trial194.stdout:(58/99): luarocks-3.9.2-5.el9.noarch.rpm 5.3 MB/s | 151 kB 00:00 2026-02-22T22:32:21.452 INFO:teuthology.orchestra.run.trial198.stdout:(57/99): grpc-data-1.46.7-10.el9.noarch.rpm 510 kB/s | 19 kB 00:00 2026-02-22T22:32:21.479 INFO:teuthology.orchestra.run.trial194.stdout:(59/99): python3-asyncssh-2.13.2-5.el9.noarch.r 18 MB/s | 548 kB 00:00 2026-02-22T22:32:21.496 INFO:teuthology.orchestra.run.trial198.stdout:(58/99): luarocks-3.9.2-5.el9.noarch.rpm 3.4 MB/s | 151 kB 00:00 2026-02-22T22:32:21.505 INFO:teuthology.orchestra.run.trial194.stdout:(60/99): python3-autocommand-2.2.2-8.el9.noarch 1.1 MB/s | 29 kB 00:00 2026-02-22T22:32:21.533 INFO:teuthology.orchestra.run.trial194.stdout:(61/99): python3-backports-tarfile-1.2.0-1.el9. 2.2 MB/s | 60 kB 00:00 2026-02-22T22:32:21.559 INFO:teuthology.orchestra.run.trial194.stdout:(62/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 1.6 MB/s | 43 kB 00:00 2026-02-22T22:32:21.560 INFO:teuthology.orchestra.run.trial198.stdout:(59/99): python3-asyncssh-2.13.2-5.el9.noarch.r 8.5 MB/s | 548 kB 00:00 2026-02-22T22:32:21.586 INFO:teuthology.orchestra.run.trial194.stdout:(63/99): python3-cachetools-4.2.4-1.el9.noarch. 1.2 MB/s | 32 kB 00:00 2026-02-22T22:32:21.597 INFO:teuthology.orchestra.run.trial198.stdout:(60/99): python3-autocommand-2.2.2-8.el9.noarch 774 kB/s | 29 kB 00:00 2026-02-22T22:32:21.612 INFO:teuthology.orchestra.run.trial194.stdout:(64/99): python3-certifi-2023.05.07-4.el9.noarc 538 kB/s | 14 kB 00:00 2026-02-22T22:32:21.629 INFO:teuthology.orchestra.run.trial198.stdout:(61/99): python3-numpy-1.23.5-2.el9.x86_64.rpm 1.8 MB/s | 6.1 MB 00:03 2026-02-22T22:32:21.636 INFO:teuthology.orchestra.run.trial198.stdout:(62/99): python3-backports-tarfile-1.2.0-1.el9. 1.5 MB/s | 60 kB 00:00 2026-02-22T22:32:21.641 INFO:teuthology.orchestra.run.trial194.stdout:(65/99): python3-cheroot-10.0.1-4.el9.noarch.rp 6.0 MB/s | 173 kB 00:00 2026-02-22T22:32:21.674 INFO:teuthology.orchestra.run.trial194.stdout:(66/99): python3-cherrypy-18.6.1-2.el9.noarch.r 11 MB/s | 358 kB 00:00 2026-02-22T22:32:21.674 INFO:teuthology.orchestra.run.trial198.stdout:(63/99): python3-cachetools-4.2.4-1.el9.noarch. 845 kB/s | 32 kB 00:00 2026-02-22T22:32:21.702 INFO:teuthology.orchestra.run.trial194.stdout:(67/99): python3-google-auth-2.45.0-1.el9.noarc 8.7 MB/s | 254 kB 00:00 2026-02-22T22:32:21.712 INFO:teuthology.orchestra.run.trial198.stdout:(64/99): python3-certifi-2023.05.07-4.el9.noarc 375 kB/s | 14 kB 00:00 2026-02-22T22:32:21.756 INFO:teuthology.orchestra.run.trial198.stdout:(65/99): python3-cheroot-10.0.1-4.el9.noarch.rp 3.8 MB/s | 173 kB 00:00 2026-02-22T22:32:21.789 INFO:teuthology.orchestra.run.trial194.stdout:(68/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 23 MB/s | 2.0 MB 00:00 2026-02-22T22:32:21.805 INFO:teuthology.orchestra.run.trial198.stdout:(66/99): python3-bcrypt-3.2.2-1.el9.x86_64.rpm 247 kB/s | 43 kB 00:00 2026-02-22T22:32:21.808 INFO:teuthology.orchestra.run.trial198.stdout:(67/99): python3-cherrypy-18.6.1-2.el9.noarch.r 6.8 MB/s | 358 kB 00:00 2026-02-22T22:32:21.818 INFO:teuthology.orchestra.run.trial194.stdout:(69/99): python3-grpcio-tools-1.46.7-10.el9.x86 5.0 MB/s | 144 kB 00:00 2026-02-22T22:32:21.846 INFO:teuthology.orchestra.run.trial194.stdout:(70/99): python3-influxdb-5.3.1-1.el9.noarch.rp 4.8 MB/s | 139 kB 00:00 2026-02-22T22:32:21.873 INFO:teuthology.orchestra.run.trial194.stdout:(71/99): python3-isodate-0.6.1-3.el9.noarch.rpm 2.0 MB/s | 56 kB 00:00 2026-02-22T22:32:21.899 INFO:teuthology.orchestra.run.trial194.stdout:(72/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 410 kB/s | 11 kB 00:00 2026-02-22T22:32:21.923 INFO:teuthology.orchestra.run.trial198.stdout:(68/99): python3-grpcio-1.46.7-10.el9.x86_64.rp 18 MB/s | 2.0 MB 00:00 2026-02-22T22:32:21.925 INFO:teuthology.orchestra.run.trial194.stdout:(73/99): python3-jaraco-classes-3.2.1-5.el9.noa 671 kB/s | 18 kB 00:00 2026-02-22T22:32:21.946 INFO:teuthology.orchestra.run.trial198.stdout:(69/99): python3-google-auth-2.45.0-1.el9.noarc 1.8 MB/s | 254 kB 00:00 2026-02-22T22:32:21.952 INFO:teuthology.orchestra.run.trial194.stdout:(74/99): python3-jaraco-collections-3.0.0-8.el9 881 kB/s | 23 kB 00:00 2026-02-22T22:32:21.963 INFO:teuthology.orchestra.run.trial198.stdout:(70/99): python3-grpcio-tools-1.46.7-10.el9.x86 3.6 MB/s | 144 kB 00:00 2026-02-22T22:32:21.979 INFO:teuthology.orchestra.run.trial194.stdout:(75/99): python3-jaraco-context-6.0.1-3.el9.noa 745 kB/s | 20 kB 00:00 2026-02-22T22:32:21.996 INFO:teuthology.orchestra.run.trial198.stdout:(71/99): python3-influxdb-5.3.1-1.el9.noarch.rp 2.7 MB/s | 139 kB 00:00 2026-02-22T22:32:22.001 INFO:teuthology.orchestra.run.trial198.stdout:(72/99): python3-isodate-0.6.1-3.el9.noarch.rpm 1.4 MB/s | 56 kB 00:00 2026-02-22T22:32:22.005 INFO:teuthology.orchestra.run.trial194.stdout:(76/99): python3-jaraco-functools-3.5.0-2.el9.n 725 kB/s | 19 kB 00:00 2026-02-22T22:32:22.032 INFO:teuthology.orchestra.run.trial194.stdout:(77/99): python3-jaraco-text-4.0.0-2.el9.noarch 1.0 MB/s | 26 kB 00:00 2026-02-22T22:32:22.035 INFO:teuthology.orchestra.run.trial198.stdout:(73/99): python3-jaraco-8.2.1-3.el9.noarch.rpm 277 kB/s | 11 kB 00:00 2026-02-22T22:32:22.039 INFO:teuthology.orchestra.run.trial198.stdout:(74/99): python3-jaraco-classes-3.2.1-5.el9.noa 466 kB/s | 18 kB 00:00 2026-02-22T22:32:22.062 INFO:teuthology.orchestra.run.trial194.stdout:(78/99): python3-kubernetes-26.1.0-3.el9.noarch 34 MB/s | 1.0 MB 00:00 2026-02-22T22:32:22.074 INFO:teuthology.orchestra.run.trial198.stdout:(75/99): python3-jaraco-collections-3.0.0-8.el9 595 kB/s | 23 kB 00:00 2026-02-22T22:32:22.076 INFO:teuthology.orchestra.run.trial198.stdout:(76/99): python3-jaraco-context-6.0.1-3.el9.noa 521 kB/s | 20 kB 00:00 2026-02-22T22:32:22.089 INFO:teuthology.orchestra.run.trial194.stdout:(79/99): python3-logutils-0.3.5-21.el9.noarch.r 1.7 MB/s | 46 kB 00:00 2026-02-22T22:32:22.105 INFO:teuthology.orchestra.run.trial194.stdout:(80/99): protobuf-compiler-3.14.0-17.el9.x86_64 838 kB/s | 862 kB 00:01 2026-02-22T22:32:22.112 INFO:teuthology.orchestra.run.trial198.stdout:(77/99): python3-jaraco-functools-3.5.0-2.el9.n 509 kB/s | 19 kB 00:00 2026-02-22T22:32:22.114 INFO:teuthology.orchestra.run.trial198.stdout:(78/99): python3-jaraco-text-4.0.0-2.el9.noarch 700 kB/s | 26 kB 00:00 2026-02-22T22:32:22.116 INFO:teuthology.orchestra.run.trial194.stdout:(81/99): python3-more-itertools-8.12.0-2.el9.no 2.8 MB/s | 79 kB 00:00 2026-02-22T22:32:22.144 INFO:teuthology.orchestra.run.trial194.stdout:(82/99): python3-natsort-7.1.1-5.el9.noarch.rpm 2.1 MB/s | 58 kB 00:00 2026-02-22T22:32:22.153 INFO:teuthology.orchestra.run.trial198.stdout:(79/99): python3-logutils-0.3.5-21.el9.noarch.r 1.2 MB/s | 46 kB 00:00 2026-02-22T22:32:22.172 INFO:teuthology.orchestra.run.trial194.stdout:(83/99): python3-pecan-1.4.2-3.el9.noarch.rpm 9.2 MB/s | 272 kB 00:00 2026-02-22T22:32:22.190 INFO:teuthology.orchestra.run.trial198.stdout:(80/99): python3-more-itertools-8.12.0-2.el9.no 2.0 MB/s | 79 kB 00:00 2026-02-22T22:32:22.197 INFO:teuthology.orchestra.run.trial194.stdout:(84/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 937 kB/s | 86 kB 00:00 2026-02-22T22:32:22.198 INFO:teuthology.orchestra.run.trial194.stdout:(85/99): python3-portend-3.1.0-2.el9.noarch.rpm 632 kB/s | 16 kB 00:00 2026-02-22T22:32:22.224 INFO:teuthology.orchestra.run.trial194.stdout:(86/99): python3-repoze-lru-0.7-16.el9.noarch.r 1.2 MB/s | 31 kB 00:00 2026-02-22T22:32:22.229 INFO:teuthology.orchestra.run.trial198.stdout:(81/99): python3-msgpack-1.0.3-2.el9.x86_64.rpm 2.2 MB/s | 86 kB 00:00 2026-02-22T22:32:22.231 INFO:teuthology.orchestra.run.trial198.stdout:(82/99): python3-kubernetes-26.1.0-3.el9.noarch 8.6 MB/s | 1.0 MB 00:00 2026-02-22T22:32:22.245 INFO:teuthology.orchestra.run.trial194.stdout:(87/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 1.8 MB/s | 90 kB 00:00 2026-02-22T22:32:22.252 INFO:teuthology.orchestra.run.trial194.stdout:(88/99): python3-routes-2.5.1-5.el9.noarch.rpm 6.7 MB/s | 188 kB 00:00 2026-02-22T22:32:22.267 INFO:teuthology.orchestra.run.trial198.stdout:(83/99): python3-natsort-7.1.1-5.el9.noarch.rpm 1.5 MB/s | 58 kB 00:00 2026-02-22T22:32:22.271 INFO:teuthology.orchestra.run.trial194.stdout:(89/99): python3-rsa-4.9-2.el9.noarch.rpm 2.3 MB/s | 59 kB 00:00 2026-02-22T22:32:22.275 INFO:teuthology.orchestra.run.trial198.stdout:(84/99): python3-pecan-1.4.2-3.el9.noarch.rpm 6.1 MB/s | 272 kB 00:00 2026-02-22T22:32:22.280 INFO:teuthology.orchestra.run.trial194.stdout:(90/99): python3-saml-1.16.0-1.el9.noarch.rpm 4.4 MB/s | 125 kB 00:00 2026-02-22T22:32:22.297 INFO:teuthology.orchestra.run.trial194.stdout:(91/99): python3-tempora-5.0.0-2.el9.noarch.rpm 1.3 MB/s | 36 kB 00:00 2026-02-22T22:32:22.304 INFO:teuthology.orchestra.run.trial198.stdout:(85/99): python3-portend-3.1.0-2.el9.noarch.rpm 436 kB/s | 16 kB 00:00 2026-02-22T22:32:22.306 INFO:teuthology.orchestra.run.trial194.stdout:(92/99): python3-typing-extensions-4.15.0-1.el9 3.2 MB/s | 86 kB 00:00 2026-02-22T22:32:22.314 INFO:teuthology.orchestra.run.trial198.stdout:(86/99): python3-pyOpenSSL-21.0.0-1.el9.noarch. 2.2 MB/s | 90 kB 00:00 2026-02-22T22:32:22.334 INFO:teuthology.orchestra.run.trial194.stdout:(93/99): python3-websocket-client-1.2.3-2.el9.n 3.2 MB/s | 90 kB 00:00 2026-02-22T22:32:22.342 INFO:teuthology.orchestra.run.trial198.stdout:(87/99): python3-repoze-lru-0.7-16.el9.noarch.r 815 kB/s | 31 kB 00:00 2026-02-22T22:32:22.347 INFO:teuthology.orchestra.run.trial194.stdout:(94/99): python3-webob-1.8.8-2.el9.noarch.rpm 4.5 MB/s | 230 kB 00:00 2026-02-22T22:32:22.356 INFO:teuthology.orchestra.run.trial198.stdout:(88/99): python3-routes-2.5.1-5.el9.noarch.rpm 4.4 MB/s | 188 kB 00:00 2026-02-22T22:32:22.362 INFO:teuthology.orchestra.run.trial194.stdout:(95/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 15 MB/s | 427 kB 00:00 2026-02-22T22:32:22.372 INFO:teuthology.orchestra.run.trial194.stdout:(96/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 1.9 MB/s | 48 kB 00:00 2026-02-22T22:32:22.381 INFO:teuthology.orchestra.run.trial198.stdout:(89/99): python3-rsa-4.9-2.el9.noarch.rpm 1.5 MB/s | 59 kB 00:00 2026-02-22T22:32:22.389 INFO:teuthology.orchestra.run.trial194.stdout:(97/99): python3-xmltodict-0.12.0-15.el9.noarch 825 kB/s | 22 kB 00:00 2026-02-22T22:32:22.396 INFO:teuthology.orchestra.run.trial198.stdout:(90/99): python3-saml-1.16.0-1.el9.noarch.rpm 3.0 MB/s | 125 kB 00:00 2026-02-22T22:32:22.398 INFO:teuthology.orchestra.run.trial194.stdout:(98/99): python3-zc-lockfile-2.0-10.el9.noarch. 763 kB/s | 20 kB 00:00 2026-02-22T22:32:22.418 INFO:teuthology.orchestra.run.trial198.stdout:(91/99): python3-tempora-5.0.0-2.el9.noarch.rpm 947 kB/s | 36 kB 00:00 2026-02-22T22:32:22.436 INFO:teuthology.orchestra.run.trial198.stdout:(92/99): python3-typing-extensions-4.15.0-1.el9 2.2 MB/s | 86 kB 00:00 2026-02-22T22:32:22.459 INFO:teuthology.orchestra.run.trial198.stdout:(93/99): python3-webob-1.8.8-2.el9.noarch.rpm 5.6 MB/s | 230 kB 00:00 2026-02-22T22:32:22.476 INFO:teuthology.orchestra.run.trial198.stdout:(94/99): python3-websocket-client-1.2.3-2.el9.n 2.2 MB/s | 90 kB 00:00 2026-02-22T22:32:22.502 INFO:teuthology.orchestra.run.trial198.stdout:(95/99): python3-werkzeug-2.0.3-3.el9.1.noarch. 9.8 MB/s | 427 kB 00:00 2026-02-22T22:32:22.513 INFO:teuthology.orchestra.run.trial198.stdout:(96/99): python3-xmlsec-1.3.13-1.el9.x86_64.rpm 1.2 MB/s | 48 kB 00:00 2026-02-22T22:32:22.539 INFO:teuthology.orchestra.run.trial198.stdout:(97/99): python3-xmltodict-0.12.0-15.el9.noarch 589 kB/s | 22 kB 00:00 2026-02-22T22:32:22.552 INFO:teuthology.orchestra.run.trial198.stdout:(98/99): python3-zc-lockfile-2.0-10.el9.noarch. 515 kB/s | 20 kB 00:00 2026-02-22T22:32:22.586 INFO:teuthology.orchestra.run.trial198.stdout:(99/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 6.1 MB/s | 19 MB 00:03 2026-02-22T22:32:22.591 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:22.591 INFO:teuthology.orchestra.run.trial198.stdout:Total 7.6 MB/s | 95 MB 00:12 2026-02-22T22:32:23.022 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:23.060 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:23.060 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:23.115 INFO:teuthology.orchestra.run.trial194.stdout:(99/99): python3-scipy-1.9.3-2.el9.x86_64.rpm 7.7 MB/s | 19 MB 00:02 2026-02-22T22:32:23.118 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:23.118 INFO:teuthology.orchestra.run.trial194.stdout:Total 6.4 MB/s | 95 MB 00:14 2026-02-22T22:32:23.477 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:23.477 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:23.527 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:23.564 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:23.565 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:23.948 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:23.950 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T22:32:23.977 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:23.977 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:23.981 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T22:32:23.987 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T22:32:24.009 INFO:teuthology.orchestra.run.trial198.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T22:32:24.014 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T22:32:24.048 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T22:32:24.057 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T22:32:24.063 INFO:teuthology.orchestra.run.trial198.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T22:32:24.066 INFO:teuthology.orchestra.run.trial198.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T22:32:24.070 INFO:teuthology.orchestra.run.trial198.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T22:32:24.093 INFO:teuthology.orchestra.run.trial198.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T22:32:24.106 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T22:32:24.109 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T22:32:24.115 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T22:32:24.117 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T22:32:24.128 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T22:32:24.134 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T22:32:24.158 INFO:teuthology.orchestra.run.trial198.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T22:32:24.162 INFO:teuthology.orchestra.run.trial198.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T22:32:24.169 INFO:teuthology.orchestra.run.trial198.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T22:32:24.193 INFO:teuthology.orchestra.run.trial198.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T22:32:24.245 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T22:32:24.258 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T22:32:24.260 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T22:32:24.265 INFO:teuthology.orchestra.run.trial198.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T22:32:24.270 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T22:32:24.275 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T22:32:24.278 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T22:32:24.292 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T22:32:24.313 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T22:32:24.318 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T22:32:24.323 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T22:32:24.334 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T22:32:24.343 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T22:32:24.350 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T22:32:24.369 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T22:32:24.378 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T22:32:24.433 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T22:32:24.443 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:24.445 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-more-itertools-8.12.0-2.el9.noarch 1/99 2026-02-22T22:32:24.445 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T22:32:24.461 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T22:32:24.465 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T22:32:24.473 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T22:32:24.475 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-jaraco-8.2.1-3.el9.noarch 2/99 2026-02-22T22:32:24.480 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-werkzeug-2.0.3-3.el9.1.noarch 3/99 2026-02-22T22:32:24.502 INFO:teuthology.orchestra.run.trial194.stdout: Installing : xmlsec1-1.2.29-13.el9.x86_64 4/99 2026-02-22T22:32:24.508 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-pyasn1-0.4.8-6.el9.noarch 5/99 2026-02-22T22:32:24.513 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T22:32:24.541 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-markupsafe-1.1.1-12.el9.x86_64 6/99 2026-02-22T22:32:24.550 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-lxml-4.6.5-3.el9.x86_64 7/99 2026-02-22T22:32:24.555 INFO:teuthology.orchestra.run.trial194.stdout: Installing : protobuf-3.14.0-17.el9.x86_64 8/99 2026-02-22T22:32:24.557 INFO:teuthology.orchestra.run.trial194.stdout: Installing : lua-5.4.4-4.el9.x86_64 9/99 2026-02-22T22:32:24.561 INFO:teuthology.orchestra.run.trial194.stdout: Installing : flexiblas-3.0.4-9.el9.x86_64 10/99 2026-02-22T22:32:24.584 INFO:teuthology.orchestra.run.trial194.stdout: Installing : unzip-6.0-59.el9.x86_64 11/99 2026-02-22T22:32:24.596 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-urllib3-1.26.5-7.el9.noarch 12/99 2026-02-22T22:32:24.599 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-requests-2.25.1-10.el9.noarch 13/99 2026-02-22T22:32:24.606 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libquadmath-11.5.0-14.el9.x86_64 14/99 2026-02-22T22:32:24.607 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libgfortran-11.5.0-14.el9.x86_64 15/99 2026-02-22T22:32:24.618 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ledmon-libs-1.1.0-3.el9.x86_64 16/99 2026-02-22T22:32:24.624 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-requests-oauthlib-1.3.0-12.el9.noarch 17/99 2026-02-22T22:32:24.648 INFO:teuthology.orchestra.run.trial194.stdout: Installing : zip-3.0-35.el9.x86_64 18/99 2026-02-22T22:32:24.651 INFO:teuthology.orchestra.run.trial194.stdout: Installing : luarocks-3.9.2-5.el9.noarch 19/99 2026-02-22T22:32:24.658 INFO:teuthology.orchestra.run.trial194.stdout: Installing : lua-devel-5.4.4-4.el9.x86_64 20/99 2026-02-22T22:32:24.682 INFO:teuthology.orchestra.run.trial194.stdout: Installing : protobuf-compiler-3.14.0-17.el9.x86_64 21/99 2026-02-22T22:32:24.732 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-mako-1.1.4-6.el9.noarch 22/99 2026-02-22T22:32:24.744 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-pyasn1-modules-0.4.8-6.el9.noarch 23/99 2026-02-22T22:32:24.746 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-rsa-4.9-2.el9.noarch 24/99 2026-02-22T22:32:24.751 INFO:teuthology.orchestra.run.trial194.stdout: Installing : xmlsec1-openssl-1.2.29-13.el9.x86_64 25/99 2026-02-22T22:32:24.756 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-xmlsec-1.3.13-1.el9.x86_64 26/99 2026-02-22T22:32:24.761 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-jaraco-classes-3.2.1-5.el9.noarch 27/99 2026-02-22T22:32:24.764 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-zc-lockfile-2.0-10.el9.noarch 28/99 2026-02-22T22:32:24.778 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-xmltodict-0.12.0-15.el9.noarch 29/99 2026-02-22T22:32:24.798 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-websocket-client-1.2.3-2.el9.noarch 30/99 2026-02-22T22:32:24.803 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-webob-1.8.8-2.el9.noarch 31/99 2026-02-22T22:32:24.808 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-typing-extensions-4.15.0-1.el9.noarch 32/99 2026-02-22T22:32:24.819 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-repoze-lru-0.7-16.el9.noarch 33/99 2026-02-22T22:32:24.827 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-routes-2.5.1-5.el9.noarch 34/99 2026-02-22T22:32:24.833 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-natsort-7.1.1-5.el9.noarch 35/99 2026-02-22T22:32:24.839 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T22:32:24.849 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T22:32:24.852 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-msgpack-1.0.3-2.el9.x86_64 36/99 2026-02-22T22:32:24.853 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T22:32:24.859 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T22:32:24.861 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T22:32:24.861 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-influxdb-5.3.1-1.el9.noarch 37/99 2026-02-22T22:32:24.889 INFO:teuthology.orchestra.run.trial198.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T22:32:24.914 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-logutils-0.3.5-21.el9.noarch 38/99 2026-02-22T22:32:24.926 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-pecan-1.4.2-3.el9.noarch 39/99 2026-02-22T22:32:24.934 INFO:teuthology.orchestra.run.trial198.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T22:32:24.941 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T22:32:24.942 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-isodate-0.6.1-3.el9.noarch 40/99 2026-02-22T22:32:24.945 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T22:32:24.946 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-saml-1.16.0-1.el9.noarch 41/99 2026-02-22T22:32:24.951 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T22:32:24.953 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-certifi-2023.05.07-4.el9.noarch 42/99 2026-02-22T22:32:24.954 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T22:32:24.961 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T22:32:24.965 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T22:32:24.992 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-cachetools-4.2.4-1.el9.noarch 43/99 2026-02-22T22:32:24.993 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T22:32:25.002 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T22:32:25.037 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T22:32:25.263 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T22:32:25.287 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T22:32:25.291 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T22:32:25.310 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-google-auth-1:2.45.0-1.el9.noarch 44/99 2026-02-22T22:32:25.319 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-kubernetes-1:26.1.0-3.el9.noarch 45/99 2026-02-22T22:32:25.323 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-backports-tarfile-1.2.0-1.el9.noarch 46/99 2026-02-22T22:32:25.329 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-jaraco-context-6.0.1-3.el9.noarch 47/99 2026-02-22T22:32:25.331 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-autocommand-2.2.2-8.el9.noarch 48/99 2026-02-22T22:32:25.350 INFO:teuthology.orchestra.run.trial198.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T22:32:25.352 INFO:teuthology.orchestra.run.trial198.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T22:32:25.358 INFO:teuthology.orchestra.run.trial194.stdout: Installing : grpc-data-1.46.7-10.el9.noarch 49/99 2026-02-22T22:32:25.373 INFO:teuthology.orchestra.run.trial198.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T22:32:25.402 INFO:teuthology.orchestra.run.trial194.stdout: Installing : abseil-cpp-20211102.0-4.el9.x86_64 50/99 2026-02-22T22:32:25.409 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-grpcio-1.46.7-10.el9.x86_64 51/99 2026-02-22T22:32:25.412 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-toml-0.10.2-6.el9.noarch 52/99 2026-02-22T22:32:25.418 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-jaraco-functools-3.5.0-2.el9.noarch 53/99 2026-02-22T22:32:25.421 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-jaraco-text-4.0.0-2.el9.noarch 54/99 2026-02-22T22:32:25.428 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-jaraco-collections-3.0.0-8.el9.noarch 55/99 2026-02-22T22:32:25.431 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-tempora-5.0.0-2.el9.noarch 56/99 2026-02-22T22:32:25.458 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-portend-3.1.0-2.el9.noarch 57/99 2026-02-22T22:32:25.468 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-protobuf-3.14.0-17.el9.noarch 58/99 2026-02-22T22:32:25.502 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-grpcio-tools-1.46.7-10.el9.x86_64 59/99 2026-02-22T22:32:25.702 INFO:teuthology.orchestra.run.trial198.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T22:32:25.721 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-devel-3.9.25-3.el9.x86_64 60/99 2026-02-22T22:32:25.745 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-babel-2.9.1-2.el9.noarch 61/99 2026-02-22T22:32:25.748 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-jinja2-2.11.3-8.el9.noarch 62/99 2026-02-22T22:32:25.776 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T22:32:25.806 INFO:teuthology.orchestra.run.trial194.stdout: Installing : openblas-0.3.29-1.el9.x86_64 63/99 2026-02-22T22:32:25.808 INFO:teuthology.orchestra.run.trial194.stdout: Installing : openblas-openmp-0.3.29-1.el9.x86_64 64/99 2026-02-22T22:32:25.829 INFO:teuthology.orchestra.run.trial194.stdout: Installing : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 65/99 2026-02-22T22:32:26.147 INFO:teuthology.orchestra.run.trial194.stdout: Installing : flexiblas-netlib-3.0.4-9.el9.x86_64 66/99 2026-02-22T22:32:26.218 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-numpy-1:1.23.5-2.el9.x86_64 67/99 2026-02-22T22:32:26.455 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T22:32:26.472 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T22:32:26.482 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T22:32:26.489 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T22:32:26.503 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T22:32:26.519 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T22:32:26.594 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T22:32:26.603 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T22:32:26.627 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T22:32:26.656 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T22:32:26.707 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T22:32:26.714 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T22:32:26.717 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T22:32:26.718 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T22:32:26.738 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:32:26.738 INFO:teuthology.orchestra.run.trial198.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T22:32:26.738 INFO:teuthology.orchestra.run.trial198.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T22:32:26.738 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:26.745 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:32:26.773 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:32:26.773 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T22:32:26.773 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:26.810 INFO:teuthology.orchestra.run.trial198.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T22:32:26.874 INFO:teuthology.orchestra.run.trial198.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T22:32:26.875 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 68/99 2026-02-22T22:32:26.876 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:32:26.891 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:32:26.891 INFO:teuthology.orchestra.run.trial198.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:26.891 INFO:teuthology.orchestra.run.trial198.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T22:32:26.891 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:26.891 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-scipy-1.9.3-2.el9.x86_64 69/99 2026-02-22T22:32:26.901 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-pyparsing-2.4.7-9.el9.noarch 70/99 2026-02-22T22:32:26.907 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-packaging-20.9-5.el9.noarch 71/99 2026-02-22T22:32:26.920 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-ply-3.11-14.el9.noarch 72/99 2026-02-22T22:32:26.936 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-pycparser-2.20-6.el9.noarch 73/99 2026-02-22T22:32:27.009 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-cffi-1.14.5-5.el9.x86_64 74/99 2026-02-22T22:32:27.018 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-cryptography-36.0.1-5.el9.x86_64 75/99 2026-02-22T22:32:27.041 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-pyOpenSSL-21.0.0-1.el9.noarch 76/99 2026-02-22T22:32:27.070 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-cheroot-10.0.1-4.el9.noarch 77/99 2026-02-22T22:32:27.119 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-cherrypy-18.6.1-2.el9.noarch 78/99 2026-02-22T22:32:27.125 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-asyncssh-2.13.2-5.el9.noarch 79/99 2026-02-22T22:32:27.129 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-bcrypt-3.2.2-1.el9.x86_64 80/99 2026-02-22T22:32:27.130 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libconfig-1.7.2-9.el9.x86_64 81/99 2026-02-22T22:32:27.147 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:32:27.147 INFO:teuthology.orchestra.run.trial194.stdout:Creating group 'libstoragemgmt' with GID 991. 2026-02-22T22:32:27.147 INFO:teuthology.orchestra.run.trial194.stdout:Creating user 'libstoragemgmt' (daemon account for libstoragemgmt) with UID 991 and GID 991. 2026-02-22T22:32:27.147 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:27.154 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:32:27.178 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: libstoragemgmt-1.10.1-1.el9.x86_64 82/99 2026-02-22T22:32:27.178 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/libstoragemgmt.service → /usr/lib/systemd/system/libstoragemgmt.service. 2026-02-22T22:32:27.178 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:27.211 INFO:teuthology.orchestra.run.trial194.stdout: Installing : python3-libstoragemgmt-1.10.1-1.el9.x86_64 83/99 2026-02-22T22:32:27.274 INFO:teuthology.orchestra.run.trial194.stdout: Installing : cryptsetup-2.8.1-3.el9.x86_64 84/99 2026-02-22T22:32:27.276 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:32:27.289 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 85/99 2026-02-22T22:32:27.289 INFO:teuthology.orchestra.run.trial194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:27.290 INFO:teuthology.orchestra.run.trial194.stdout:Invalid unit name "ceph-volume@*.service" escaped as "ceph-volume@\x2a.service". 2026-02-22T22:32:27.290 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:27.589 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:32:27.615 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:32:27.616 INFO:teuthology.orchestra.run.trial198.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:27.616 INFO:teuthology.orchestra.run.trial198.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T22:32:27.616 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:32:27.616 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:32:27.616 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:27.675 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:32:27.677 INFO:teuthology.orchestra.run.trial198.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:32:27.682 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T22:32:27.703 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T22:32:27.705 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:32:27.985 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:32:28.009 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 86/99 2026-02-22T22:32:28.009 INFO:teuthology.orchestra.run.trial194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:28.009 INFO:teuthology.orchestra.run.trial194.stdout:Invalid unit name "ceph-osd@*.service" escaped as "ceph-osd@\x2a.service". 2026-02-22T22:32:28.009 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:32:28.010 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target → /usr/lib/systemd/system/ceph-osd.target. 2026-02-22T22:32:28.010 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:28.066 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:32:28.069 INFO:teuthology.orchestra.run.trial194.stdout: Installing : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 87/99 2026-02-22T22:32:28.075 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 88/99 2026-02-22T22:32:28.096 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 89/99 2026-02-22T22:32:28.097 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:32:28.200 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:32:28.203 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:32:28.584 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 90/99 2026-02-22T22:32:28.586 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:32:28.661 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:32:28.662 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:32:28.676 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:32:28.676 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:32:28.731 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:32:28.781 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T22:32:28.782 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:32:28.804 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:32:28.804 INFO:teuthology.orchestra.run.trial198.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:28.804 INFO:teuthology.orchestra.run.trial198.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T22:32:28.804 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:32:28.804 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:32:28.804 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:28.815 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:32:29.033 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:32:29.035 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:32:29.039 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 91/99 2026-02-22T22:32:29.040 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:32:29.052 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 92/99 2026-02-22T22:32:29.053 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:32:29.057 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:32:29.057 INFO:teuthology.orchestra.run.trial198.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:29.057 INFO:teuthology.orchestra.run.trial198.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T22:32:29.057 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:32:29.057 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:32:29.057 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:29.108 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 93/99 2026-02-22T22:32:29.157 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 94/99 2026-02-22T22:32:29.158 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:32:29.158 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:32:29.179 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 95/99 2026-02-22T22:32:29.179 INFO:teuthology.orchestra.run.trial194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:29.179 INFO:teuthology.orchestra.run.trial194.stdout:Invalid unit name "ceph-mgr@*.service" escaped as "ceph-mgr@\x2a.service". 2026-02-22T22:32:29.179 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:32:29.179 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target → /usr/lib/systemd/system/ceph-mgr.target. 2026-02-22T22:32:29.179 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:29.180 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:32:29.180 INFO:teuthology.orchestra.run.trial198.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:29.180 INFO:teuthology.orchestra.run.trial198.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T22:32:29.181 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:32:29.181 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:32:29.181 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:29.190 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:32:29.198 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:32:29.409 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 96/99 2026-02-22T22:32:29.410 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:32:29.431 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 97/99 2026-02-22T22:32:29.431 INFO:teuthology.orchestra.run.trial194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:29.431 INFO:teuthology.orchestra.run.trial194.stdout:Invalid unit name "ceph-mon@*.service" escaped as "ceph-mon@\x2a.service". 2026-02-22T22:32:29.431 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:32:29.431 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target → /usr/lib/systemd/system/ceph-mon.target. 2026-02-22T22:32:29.432 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:29.533 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:32:29.553 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 98/99 2026-02-22T22:32:29.553 INFO:teuthology.orchestra.run.trial194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:29.553 INFO:teuthology.orchestra.run.trial194.stdout:Invalid unit name "ceph-mds@*.service" escaped as "ceph-mds@\x2a.service". 2026-02-22T22:32:29.553 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:32:29.553 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target → /usr/lib/systemd/system/ceph-mds.target. 2026-02-22T22:32:29.553 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:29.571 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:32:29.880 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:32:29.880 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T22:32:29.880 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T22:32:29.880 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T22:32:29.880 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T22:32:29.880 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T22:32:29.880 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T22:32:29.880 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T22:32:29.881 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T22:32:29.882 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T22:32:29.883 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T22:32:29.883 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T22:32:29.883 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T22:32:29.883 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T22:32:29.883 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T22:32:29.883 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T22:32:29.883 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T22:32:29.884 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T22:32:29.884 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T22:32:29.884 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T22:32:29.884 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T22:32:29.884 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T22:32:29.884 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T22:32:29.884 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T22:32:29.885 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T22:32:29.885 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T22:32:29.885 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T22:32:29.885 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T22:32:29.885 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T22:32:29.885 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T22:32:29.885 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T22:32:29.886 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T22:32:29.886 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T22:32:29.886 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T22:32:29.886 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T22:32:29.886 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T22:32:29.886 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T22:32:29.887 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T22:32:29.887 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T22:32:29.887 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T22:32:29.887 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T22:32:29.887 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T22:32:29.887 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T22:32:29.888 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T22:32:29.889 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T22:32:29.890 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T22:32:29.890 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T22:32:29.890 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T22:32:29.890 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T22:32:29.890 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T22:32:29.890 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T22:32:29.890 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.941 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T22:32:29.942 INFO:teuthology.orchestra.run.trial198.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T22:32:29.943 INFO:teuthology.orchestra.run.trial198.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: zip-3.0-35.el9.x86_64 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:29.944 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:29.997 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-base 2026-02-22T22:32:30.210 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 99/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 1/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 3/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 4/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 5/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9 6/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 7/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noar 8/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4e 9/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noar 10/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.n 11/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 12/99 2026-02-22T22:32:30.211 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9. 13/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 14/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 15/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : cryptsetup-2.8.1-3.el9.x86_64 16/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ledmon-libs-1.1.0-3.el9.x86_64 17/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libconfig-1.7.2-9.el9.x86_64 18/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libgfortran-11.5.0-14.el9.x86_64 19/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libquadmath-11.5.0-14.el9.x86_64 20/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-cffi-1.14.5-5.el9.x86_64 21/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-cryptography-36.0.1-5.el9.x86_64 22/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-ply-3.11-14.el9.noarch 23/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-pycparser-2.20-6.el9.noarch 24/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-pyparsing-2.4.7-9.el9.noarch 25/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-requests-2.25.1-10.el9.noarch 26/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-urllib3-1.26.5-7.el9.noarch 27/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : unzip-6.0-59.el9.x86_64 28/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : zip-3.0-35.el9.x86_64 29/99 2026-02-22T22:32:30.212 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : flexiblas-3.0.4-9.el9.x86_64 30/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : flexiblas-netlib-3.0.4-9.el9.x86_64 31/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 32/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libstoragemgmt-1.10.1-1.el9.x86_64 33/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : lua-5.4.4-4.el9.x86_64 34/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : openblas-0.3.29-1.el9.x86_64 35/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : openblas-openmp-0.3.29-1.el9.x86_64 36/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : protobuf-3.14.0-17.el9.x86_64 37/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-babel-2.9.1-2.el9.noarch 38/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-devel-3.9.25-3.el9.x86_64 39/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-jinja2-2.11.3-8.el9.noarch 40/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-libstoragemgmt-1.10.1-1.el9.x86_64 41/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-lxml-4.6.5-3.el9.x86_64 42/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-mako-1.1.4-6.el9.noarch 43/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-markupsafe-1.1.1-12.el9.x86_64 44/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-numpy-1:1.23.5-2.el9.x86_64 45/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-numpy-f2py-1:1.23.5-2.el9.x86_64 46/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-packaging-20.9-5.el9.noarch 47/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-protobuf-3.14.0-17.el9.noarch 48/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-pyasn1-0.4.8-6.el9.noarch 49/99 2026-02-22T22:32:30.213 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-pyasn1-modules-0.4.8-6.el9.noarch 50/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-requests-oauthlib-1.3.0-12.el9.noarch 51/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-scipy-1.9.3-2.el9.x86_64 52/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-toml-0.10.2-6.el9.noarch 53/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : xmlsec1-1.2.29-13.el9.x86_64 54/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : xmlsec1-openssl-1.2.29-13.el9.x86_64 55/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : lua-devel-5.4.4-4.el9.x86_64 56/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : protobuf-compiler-3.14.0-17.el9.x86_64 57/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : abseil-cpp-20211102.0-4.el9.x86_64 58/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : grpc-data-1.46.7-10.el9.noarch 59/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : luarocks-3.9.2-5.el9.noarch 60/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-asyncssh-2.13.2-5.el9.noarch 61/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-autocommand-2.2.2-8.el9.noarch 62/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-backports-tarfile-1.2.0-1.el9.noarch 63/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-bcrypt-3.2.2-1.el9.x86_64 64/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-cachetools-4.2.4-1.el9.noarch 65/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-certifi-2023.05.07-4.el9.noarch 66/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-cheroot-10.0.1-4.el9.noarch 67/99 2026-02-22T22:32:30.214 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-cherrypy-18.6.1-2.el9.noarch 68/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-google-auth-1:2.45.0-1.el9.noarch 69/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-grpcio-1.46.7-10.el9.x86_64 70/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-grpcio-tools-1.46.7-10.el9.x86_64 71/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-influxdb-5.3.1-1.el9.noarch 72/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-isodate-0.6.1-3.el9.noarch 73/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-jaraco-8.2.1-3.el9.noarch 74/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-jaraco-classes-3.2.1-5.el9.noarch 75/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-jaraco-collections-3.0.0-8.el9.noarch 76/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-jaraco-context-6.0.1-3.el9.noarch 77/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-jaraco-functools-3.5.0-2.el9.noarch 78/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-jaraco-text-4.0.0-2.el9.noarch 79/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-kubernetes-1:26.1.0-3.el9.noarch 80/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-logutils-0.3.5-21.el9.noarch 81/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-more-itertools-8.12.0-2.el9.noarch 82/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-msgpack-1.0.3-2.el9.x86_64 83/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-natsort-7.1.1-5.el9.noarch 84/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-pecan-1.4.2-3.el9.noarch 85/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-portend-3.1.0-2.el9.noarch 86/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-pyOpenSSL-21.0.0-1.el9.noarch 87/99 2026-02-22T22:32:30.215 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-repoze-lru-0.7-16.el9.noarch 88/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-routes-2.5.1-5.el9.noarch 89/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-rsa-4.9-2.el9.noarch 90/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-saml-1.16.0-1.el9.noarch 91/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-tempora-5.0.0-2.el9.noarch 92/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-typing-extensions-4.15.0-1.el9.noarch 93/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-webob-1.8.8-2.el9.noarch 94/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-websocket-client-1.2.3-2.el9.noarch 95/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-werkzeug-2.0.3-3.el9.1.noarch 96/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-xmlsec-1.3.13-1.el9.x86_64 97/99 2026-02-22T22:32:30.216 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-xmltodict-0.12.0-15.el9.noarch 98/99 2026-02-22T22:32:30.276 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : python3-zc-lockfile-2.0-10.el9.noarch 99/99 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: abseil-cpp-20211102.0-4.el9.x86_64 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: ceph-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: ceph-grafana-dashboards-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mds-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.277 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-k8sevents-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-modules-core-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: ceph-mon-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: ceph-osd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: ceph-prometheus-alerts-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: cephadm-2:19.2.3-689.gf90f4eae.el9.noarch 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: cryptsetup-2.8.1-3.el9.x86_64 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: flexiblas-3.0.4-9.el9.x86_64 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: flexiblas-netlib-3.0.4-9.el9.x86_64 2026-02-22T22:32:30.278 INFO:teuthology.orchestra.run.trial194.stdout: flexiblas-openblas-openmp-3.0.4-9.el9.x86_64 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: grpc-data-1.46.7-10.el9.noarch 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: ledmon-libs-1.1.0-3.el9.x86_64 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: libconfig-1.7.2-9.el9.x86_64 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: libgfortran-11.5.0-14.el9.x86_64 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: libquadmath-11.5.0-14.el9.x86_64 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: lua-5.4.4-4.el9.x86_64 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: lua-devel-5.4.4-4.el9.x86_64 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: luarocks-3.9.2-5.el9.noarch 2026-02-22T22:32:30.279 INFO:teuthology.orchestra.run.trial194.stdout: openblas-0.3.29-1.el9.x86_64 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: openblas-openmp-0.3.29-1.el9.x86_64 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: protobuf-3.14.0-17.el9.x86_64 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: protobuf-compiler-3.14.0-17.el9.x86_64 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-asyncssh-2.13.2-5.el9.noarch 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-autocommand-2.2.2-8.el9.noarch 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-babel-2.9.1-2.el9.noarch 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-backports-tarfile-1.2.0-1.el9.noarch 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-bcrypt-3.2.2-1.el9.x86_64 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-cachetools-4.2.4-1.el9.noarch 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-certifi-2023.05.07-4.el9.noarch 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-cffi-1.14.5-5.el9.x86_64 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-cheroot-10.0.1-4.el9.noarch 2026-02-22T22:32:30.280 INFO:teuthology.orchestra.run.trial194.stdout: python3-cherrypy-18.6.1-2.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-cryptography-36.0.1-5.el9.x86_64 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-devel-3.9.25-3.el9.x86_64 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-google-auth-1:2.45.0-1.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-grpcio-1.46.7-10.el9.x86_64 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-grpcio-tools-1.46.7-10.el9.x86_64 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-influxdb-5.3.1-1.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-isodate-0.6.1-3.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-8.2.1-3.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-classes-3.2.1-5.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-collections-3.0.0-8.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-context-6.0.1-3.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-functools-3.5.0-2.el9.noarch 2026-02-22T22:32:30.281 INFO:teuthology.orchestra.run.trial194.stdout: python3-jaraco-text-4.0.0-2.el9.noarch 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-jinja2-2.11.3-8.el9.noarch 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-kubernetes-1:26.1.0-3.el9.noarch 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-libstoragemgmt-1.10.1-1.el9.x86_64 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-logutils-0.3.5-21.el9.noarch 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-lxml-4.6.5-3.el9.x86_64 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-mako-1.1.4-6.el9.noarch 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-markupsafe-1.1.1-12.el9.x86_64 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-more-itertools-8.12.0-2.el9.noarch 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-msgpack-1.0.3-2.el9.x86_64 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-natsort-7.1.1-5.el9.noarch 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-numpy-1:1.23.5-2.el9.x86_64 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-numpy-f2py-1:1.23.5-2.el9.x86_64 2026-02-22T22:32:30.282 INFO:teuthology.orchestra.run.trial194.stdout: python3-packaging-20.9-5.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-pecan-1.4.2-3.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-ply-3.11-14.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-portend-3.1.0-2.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-protobuf-3.14.0-17.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-pyOpenSSL-21.0.0-1.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-pyasn1-0.4.8-6.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-pyasn1-modules-0.4.8-6.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-pycparser-2.20-6.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-pyparsing-2.4.7-9.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-repoze-lru-0.7-16.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-requests-2.25.1-10.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-requests-oauthlib-1.3.0-12.el9.noarch 2026-02-22T22:32:30.283 INFO:teuthology.orchestra.run.trial194.stdout: python3-routes-2.5.1-5.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-rsa-4.9-2.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-saml-1.16.0-1.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-scipy-1.9.3-2.el9.x86_64 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-tempora-5.0.0-2.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-toml-0.10.2-6.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-typing-extensions-4.15.0-1.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-urllib3-1.26.5-7.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-webob-1.8.8-2.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-websocket-client-1.2.3-2.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-werkzeug-2.0.3-3.el9.1.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-xmlsec-1.3.13-1.el9.x86_64 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-xmltodict-0.12.0-15.el9.noarch 2026-02-22T22:32:30.284 INFO:teuthology.orchestra.run.trial194.stdout: python3-zc-lockfile-2.0-10.el9.noarch 2026-02-22T22:32:30.285 INFO:teuthology.orchestra.run.trial194.stdout: unzip-6.0-59.el9.x86_64 2026-02-22T22:32:30.285 INFO:teuthology.orchestra.run.trial194.stdout: xmlsec1-1.2.29-13.el9.x86_64 2026-02-22T22:32:30.285 INFO:teuthology.orchestra.run.trial194.stdout: xmlsec1-openssl-1.2.29-13.el9.x86_64 2026-02-22T22:32:30.285 INFO:teuthology.orchestra.run.trial194.stdout: zip-3.0-35.el9.x86_64 2026-02-22T22:32:30.285 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:30.285 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:30.312 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:30.332 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-base 2026-02-22T22:32:30.378 INFO:teuthology.orchestra.run.trial198.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:30.396 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:30.397 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:30.397 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:30.418 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install cephadm 2026-02-22T22:32:30.644 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:30.710 INFO:teuthology.orchestra.run.trial194.stdout:Package ceph-base-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:30.728 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:30.729 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:30.729 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:30.731 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:39 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:30.750 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install cephadm 2026-02-22T22:32:30.797 INFO:teuthology.orchestra.run.trial198.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:30.816 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:30.817 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:30.817 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:30.838 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T22:32:31.058 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:31.123 INFO:teuthology.orchestra.run.trial194.stdout:Package cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:31.141 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:31.142 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:31.142 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:31.147 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:31.162 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-immutable-object-cache 2026-02-22T22:32:31.233 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:31.233 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:31.233 INFO:teuthology.orchestra.run.trial198.stdout: Package Arch Version Repo Size 2026-02-22T22:32:31.233 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:31.233 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:31.233 INFO:teuthology.orchestra.run.trial198.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T22:32:31.233 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:31.234 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:31.234 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:31.234 INFO:teuthology.orchestra.run.trial198.stdout:Install 1 Package 2026-02-22T22:32:31.234 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:31.234 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 145 k 2026-02-22T22:32:31.234 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 431 k 2026-02-22T22:32:31.234 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:31.358 INFO:teuthology.orchestra.run.trial198.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.1 MB/s | 145 kB 00:00 2026-02-22T22:32:31.358 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:31.358 INFO:teuthology.orchestra.run.trial198.stdout:Total 1.1 MB/s | 145 kB 00:00 2026-02-22T22:32:31.358 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:31.362 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:31.362 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:31.393 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:31.393 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:31.449 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:31.450 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:32:31.469 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:40 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:31.473 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:32:31.473 INFO:teuthology.orchestra.run.trial198.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:31.473 INFO:teuthology.orchestra.run.trial198.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T22:32:31.473 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:31.552 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:31.552 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:31.552 INFO:teuthology.orchestra.run.trial194.stdout: Package Arch Version Repo Size 2026-02-22T22:32:31.552 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:31.552 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:31.552 INFO:teuthology.orchestra.run.trial194.stdout: ceph-immutable-object-cache x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 145 k 2026-02-22T22:32:31.552 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:31.553 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:31.553 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:31.553 INFO:teuthology.orchestra.run.trial194.stdout:Install 1 Package 2026-02-22T22:32:31.553 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:31.553 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 145 k 2026-02-22T22:32:31.553 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 431 k 2026-02-22T22:32:31.553 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:31.681 INFO:teuthology.orchestra.run.trial194.stdout:ceph-immutable-object-cache-19.2.3-689.gf90f4ea 1.1 MB/s | 145 kB 00:00 2026-02-22T22:32:31.682 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:31.682 INFO:teuthology.orchestra.run.trial194.stdout:Total 1.1 MB/s | 145 kB 00:00 2026-02-22T22:32:31.682 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:31.685 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:31.686 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:31.717 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:31.717 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:31.753 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:32:31.753 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:31.753 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:31.753 INFO:teuthology.orchestra.run.trial198.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:31.753 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:31.753 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:31.772 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:31.773 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:32:31.780 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-mgr 2026-02-22T22:32:31.795 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:32:31.795 INFO:teuthology.orchestra.run.trial194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:31.795 INFO:teuthology.orchestra.run.trial194.stdout:Invalid unit name "ceph-immutable-object-cache@*.service" escaped as "ceph-immutable-object-cache@\x2a.service". 2026-02-22T22:32:31.795 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:32.088 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:32.117 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.e 1/1 2026-02-22T22:32:32.118 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:32.118 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:32.118 INFO:teuthology.orchestra.run.trial194.stdout: ceph-immutable-object-cache-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:32.118 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:32.118 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:32.145 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-mgr 2026-02-22T22:32:32.153 INFO:teuthology.orchestra.run.trial198.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:32.172 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:32.172 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:32.173 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:32.193 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T22:32:32.456 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:32.499 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:32.520 INFO:teuthology.orchestra.run.trial194.stdout:Package ceph-mgr-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:32.539 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:32.539 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:32.539 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:32.561 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-mgr-dashboard 2026-02-22T22:32:32.564 INFO:teuthology.orchestra.run.trial198.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:32.582 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:32.582 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:32.582 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:32.603 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T22:32:32.870 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:32.907 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:41 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:32.935 INFO:teuthology.orchestra.run.trial194.stdout:Package ceph-mgr-dashboard-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:32.954 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:32.955 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:32.955 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:32.973 INFO:teuthology.orchestra.run.trial198.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:32.976 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-mgr-diskprediction-local 2026-02-22T22:32:32.991 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:32.992 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:32.992 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:33.013 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-mgr-rook 2026-02-22T22:32:33.292 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:33.318 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:33.358 INFO:teuthology.orchestra.run.trial194.stdout:Package ceph-mgr-diskprediction-local-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:33.377 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:33.377 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:33.377 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:33.382 INFO:teuthology.orchestra.run.trial198.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:33.398 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-mgr-rook 2026-02-22T22:32:33.401 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:33.401 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:33.402 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:33.423 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T22:32:33.708 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:33.726 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:42 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:33.773 INFO:teuthology.orchestra.run.trial194.stdout:Package ceph-mgr-rook-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:33.791 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:33.791 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:33.791 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:33.792 INFO:teuthology.orchestra.run.trial198.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:33.810 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:33.811 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:33.811 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:33.813 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-mgr-cephadm 2026-02-22T22:32:33.833 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-fuse 2026-02-22T22:32:34.122 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:43 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:34.140 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:43 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:34.187 INFO:teuthology.orchestra.run.trial194.stdout:Package ceph-mgr-cephadm-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:34.205 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:34.206 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:34.206 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:34.224 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:34.224 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:34.224 INFO:teuthology.orchestra.run.trial198.stdout: Package Architecture Version Repository Size 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout:Install 1 Package 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 1.2 M 2026-02-22T22:32:34.225 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 3.6 M 2026-02-22T22:32:34.226 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:34.228 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-fuse 2026-02-22T22:32:34.345 INFO:teuthology.orchestra.run.trial198.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 9.6 MB/s | 1.2 MB 00:00 2026-02-22T22:32:34.346 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:34.346 INFO:teuthology.orchestra.run.trial198.stdout:Total 9.5 MB/s | 1.2 MB 00:00 2026-02-22T22:32:34.346 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:34.350 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:34.350 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:34.387 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:34.387 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:34.494 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:34.511 INFO:teuthology.orchestra.run.trial198.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:34.541 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:43 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout: Package Architecture Version Repository Size 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout: ceph-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 1.2 M 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout:Install 1 Package 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:34.626 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 1.2 M 2026-02-22T22:32:34.627 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 3.6 M 2026-02-22T22:32:34.627 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:34.745 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:34.749 INFO:teuthology.orchestra.run.trial194.stdout:ceph-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 9.5 MB/s | 1.2 MB 00:00 2026-02-22T22:32:34.749 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:34.749 INFO:teuthology.orchestra.run.trial194.stdout:Total 9.4 MB/s | 1.2 MB 00:00 2026-02-22T22:32:34.749 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:34.753 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:34.753 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:34.789 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:34.789 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:34.802 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:34.802 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:34.802 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:34.802 INFO:teuthology.orchestra.run.trial198.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:34.802 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:34.802 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:34.830 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install ceph-volume 2026-02-22T22:32:34.892 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:34.909 INFO:teuthology.orchestra.run.trial194.stdout: Installing : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:35.128 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:35.151 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:44 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:35.184 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:35.184 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:35.184 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:35.185 INFO:teuthology.orchestra.run.trial194.stdout: ceph-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:35.185 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:35.185 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:35.212 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install ceph-volume 2026-02-22T22:32:35.216 INFO:teuthology.orchestra.run.trial198.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:35.234 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:35.235 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:35.235 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:35.257 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install librados-devel 2026-02-22T22:32:35.524 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:44 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:35.566 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:44 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:35.590 INFO:teuthology.orchestra.run.trial194.stdout:Package ceph-volume-2:19.2.3-689.gf90f4eae.el9.noarch is already installed. 2026-02-22T22:32:35.609 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:35.609 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:35.609 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:35.631 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install librados-devel 2026-02-22T22:32:35.651 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:35.651 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:35.651 INFO:teuthology.orchestra.run.trial198.stdout: Package Arch Version Repo Size 2026-02-22T22:32:35.651 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout:Install 1 Package 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 126 k 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 456 k 2026-02-22T22:32:35.652 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:35.738 INFO:teuthology.orchestra.run.trial198.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-22T22:32:35.739 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:35.739 INFO:teuthology.orchestra.run.trial198.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-22T22:32:35.739 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:35.742 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:35.742 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:35.775 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:35.775 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:35.831 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:35.848 INFO:teuthology.orchestra.run.trial198.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:35.938 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:35.944 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:44 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:35.995 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:35.996 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:35.996 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:35.996 INFO:teuthology.orchestra.run.trial198.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:35.996 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:35.996 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:36.024 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install libcephfs2 2026-02-22T22:32:36.027 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:36.028 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:36.028 INFO:teuthology.orchestra.run.trial194.stdout: Package Arch Version Repo Size 2026-02-22T22:32:36.028 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:36.028 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout: librados-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 126 k 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout:Install 1 Package 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 126 k 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 456 k 2026-02-22T22:32:36.029 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:36.116 INFO:teuthology.orchestra.run.trial194.stdout:librados-devel-19.2.3-689.gf90f4eae.el9.x86_64. 1.4 MB/s | 126 kB 00:00 2026-02-22T22:32:36.117 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:36.117 INFO:teuthology.orchestra.run.trial194.stdout:Total 1.4 MB/s | 126 kB 00:00 2026-02-22T22:32:36.117 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:36.120 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:36.120 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:36.153 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:36.153 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:36.208 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:36.224 INFO:teuthology.orchestra.run.trial194.stdout: Installing : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:36.307 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:36.334 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:45 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:36.366 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:36.366 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:36.366 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:36.366 INFO:teuthology.orchestra.run.trial194.stdout: librados-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:36.366 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:36.366 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:36.392 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install libcephfs2 2026-02-22T22:32:36.398 INFO:teuthology.orchestra.run.trial198.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:36.416 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:36.417 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:36.417 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:36.441 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install libcephfs-devel 2026-02-22T22:32:36.710 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:45 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:36.748 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:45 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:36.776 INFO:teuthology.orchestra.run.trial194.stdout:Package libcephfs2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:36.794 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:36.796 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:36.796 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:36.815 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install libcephfs-devel 2026-02-22T22:32:36.834 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:36.834 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout: Package Arch Version Repo Size 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout:Install 1 Package 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:36.835 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 34 k 2026-02-22T22:32:36.836 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 153 k 2026-02-22T22:32:36.836 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:36.899 INFO:teuthology.orchestra.run.trial198.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 519 kB/s | 34 kB 00:00 2026-02-22T22:32:36.900 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:36.900 INFO:teuthology.orchestra.run.trial198.stdout:Total 507 kB/s | 34 kB 00:00 2026-02-22T22:32:36.900 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:36.902 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:36.902 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:36.905 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:36.905 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:36.926 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:36.942 INFO:teuthology.orchestra.run.trial198.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:37.002 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:37.062 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:37.062 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:37.062 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:37.062 INFO:teuthology.orchestra.run.trial198.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:37.062 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:37.063 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:37.090 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install librados2 2026-02-22T22:32:37.125 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:46 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:37.208 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:37.208 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:37.208 INFO:teuthology.orchestra.run.trial194.stdout: Package Arch Version Repo Size 2026-02-22T22:32:37.208 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:37.208 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout: libcephfs-devel x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 34 k 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout:Install 1 Package 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 34 k 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 153 k 2026-02-22T22:32:37.209 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:37.273 INFO:teuthology.orchestra.run.trial194.stdout:libcephfs-devel-19.2.3-689.gf90f4eae.el9.x86_64 521 kB/s | 34 kB 00:00 2026-02-22T22:32:37.274 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:37.274 INFO:teuthology.orchestra.run.trial194.stdout:Total 514 kB/s | 34 kB 00:00 2026-02-22T22:32:37.274 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:37.276 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:37.276 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:37.278 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:37.278 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:37.300 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:37.316 INFO:teuthology.orchestra.run.trial194.stdout: Installing : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:37.371 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:37.398 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:46 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:37.429 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:37.429 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:37.429 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:37.429 INFO:teuthology.orchestra.run.trial194.stdout: libcephfs-devel-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:37.429 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:37.429 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:37.456 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install librados2 2026-02-22T22:32:37.464 INFO:teuthology.orchestra.run.trial198.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:37.483 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:37.483 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:37.483 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:37.505 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install librbd1 2026-02-22T22:32:37.768 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:46 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:37.813 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:46 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:37.834 INFO:teuthology.orchestra.run.trial194.stdout:Package librados2-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:37.852 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:37.853 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:37.853 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:37.874 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install librbd1 2026-02-22T22:32:37.877 INFO:teuthology.orchestra.run.trial198.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:37.895 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:37.896 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:37.896 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:37.917 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install python3-rados 2026-02-22T22:32:38.183 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:47 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:38.226 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:47 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:38.249 INFO:teuthology.orchestra.run.trial194.stdout:Package librbd1-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:38.267 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:38.268 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:38.268 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:38.289 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install python3-rados 2026-02-22T22:32:38.292 INFO:teuthology.orchestra.run.trial198.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:38.310 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:38.310 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:38.310 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:38.332 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install python3-rgw 2026-02-22T22:32:38.599 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:47 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:38.639 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:47 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:38.664 INFO:teuthology.orchestra.run.trial194.stdout:Package python3-rados-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:38.682 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:38.683 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:38.683 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:38.704 INFO:teuthology.orchestra.run.trial198.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:38.705 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install python3-rgw 2026-02-22T22:32:38.723 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:38.723 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:38.724 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:38.745 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install python3-cephfs 2026-02-22T22:32:39.018 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:48 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:39.052 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:48 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:39.085 INFO:teuthology.orchestra.run.trial194.stdout:Package python3-rgw-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:39.103 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:39.104 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:39.104 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:39.117 INFO:teuthology.orchestra.run.trial198.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:39.125 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install python3-cephfs 2026-02-22T22:32:39.136 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:39.136 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:39.136 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:39.157 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install python3-rbd 2026-02-22T22:32:39.432 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:48 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:39.465 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:48 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:39.497 INFO:teuthology.orchestra.run.trial194.stdout:Package python3-cephfs-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:39.515 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:39.516 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:39.516 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:39.529 INFO:teuthology.orchestra.run.trial198.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:39.537 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install python3-rbd 2026-02-22T22:32:39.548 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:39.549 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:39.549 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:39.571 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install rbd-fuse 2026-02-22T22:32:39.846 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:48 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:39.877 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:48 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:39.913 INFO:teuthology.orchestra.run.trial194.stdout:Package python3-rbd-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:39.931 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:39.932 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:39.932 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:39.953 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install rbd-fuse 2026-02-22T22:32:39.962 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout: Package Architecture Version Repository Size 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:39.963 INFO:teuthology.orchestra.run.trial198.stdout:Install 1 Package 2026-02-22T22:32:39.964 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:39.964 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 85 k 2026-02-22T22:32:39.964 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 227 k 2026-02-22T22:32:39.964 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:40.038 INFO:teuthology.orchestra.run.trial198.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.1 MB/s | 85 kB 00:00 2026-02-22T22:32:40.039 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:40.039 INFO:teuthology.orchestra.run.trial198.stdout:Total 1.1 MB/s | 85 kB 00:00 2026-02-22T22:32:40.039 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:40.042 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:40.042 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:40.073 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:40.073 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:40.124 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:40.141 INFO:teuthology.orchestra.run.trial198.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:40.227 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:40.264 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:49 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:40.284 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:40.284 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:40.284 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:40.285 INFO:teuthology.orchestra.run.trial198.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:40.285 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:40.285 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:40.313 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install rbd-mirror 2026-02-22T22:32:40.348 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout: Package Architecture Version Repository Size 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout: rbd-fuse x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 85 k 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:40.349 INFO:teuthology.orchestra.run.trial194.stdout:Install 1 Package 2026-02-22T22:32:40.350 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:40.350 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 85 k 2026-02-22T22:32:40.350 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 227 k 2026-02-22T22:32:40.350 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:40.547 INFO:teuthology.orchestra.run.trial194.stdout:rbd-fuse-19.2.3-689.gf90f4eae.el9.x86_64.rpm 431 kB/s | 85 kB 00:00 2026-02-22T22:32:40.547 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:40.547 INFO:teuthology.orchestra.run.trial194.stdout:Total 429 kB/s | 85 kB 00:00 2026-02-22T22:32:40.547 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:40.550 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:40.550 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:40.581 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:40.582 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:40.626 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:49 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:40.632 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:40.649 INFO:teuthology.orchestra.run.trial194.stdout: Installing : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:40.710 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:40.710 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:40.710 INFO:teuthology.orchestra.run.trial198.stdout: Package Arch Version Repo Size 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout:Install 1 Package 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 3.1 M 2026-02-22T22:32:40.711 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 13 M 2026-02-22T22:32:40.712 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:40.728 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:40.787 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:40.787 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:40.787 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:40.787 INFO:teuthology.orchestra.run.trial194.stdout: rbd-fuse-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:40.787 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:40.787 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:40.816 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install rbd-mirror 2026-02-22T22:32:40.863 INFO:teuthology.orchestra.run.trial198.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 20 MB/s | 3.1 MB 00:00 2026-02-22T22:32:40.864 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:40.864 INFO:teuthology.orchestra.run.trial198.stdout:Total 20 MB/s | 3.1 MB 00:00 2026-02-22T22:32:40.864 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:40.868 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:40.868 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:40.909 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:40.909 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:41.103 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:41.104 INFO:teuthology.orchestra.run.trial198.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:41.128 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:41.129 INFO:teuthology.orchestra.run.trial198.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:41.129 INFO:teuthology.orchestra.run.trial198.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T22:32:41.129 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:32:41.129 INFO:teuthology.orchestra.run.trial198.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:32:41.129 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:41.135 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:50 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:41.220 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:41.220 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:41.220 INFO:teuthology.orchestra.run.trial194.stdout: Package Arch Version Repo Size 2026-02-22T22:32:41.220 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:41.220 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout: rbd-mirror x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 3.1 M 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout:Install 1 Package 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 3.1 M 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 13 M 2026-02-22T22:32:41.221 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:41.410 INFO:teuthology.orchestra.run.trial194.stdout:rbd-mirror-19.2.3-689.gf90f4eae.el9.x86_64.rpm 16 MB/s | 3.1 MB 00:00 2026-02-22T22:32:41.410 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:41.410 INFO:teuthology.orchestra.run.trial194.stdout:Total 16 MB/s | 3.1 MB 00:00 2026-02-22T22:32:41.410 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:41.414 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:41.414 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:41.429 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:41.429 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:41.430 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:41.430 INFO:teuthology.orchestra.run.trial198.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:41.430 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:41.430 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:41.454 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:41.455 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:41.459 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install rbd-nbd 2026-02-22T22:32:41.648 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:41.650 INFO:teuthology.orchestra.run.trial194.stdout: Installing : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:41.673 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:41.673 INFO:teuthology.orchestra.run.trial194.stdout:Glob pattern passed to enable, but globs are not supported for this. 2026-02-22T22:32:41.673 INFO:teuthology.orchestra.run.trial194.stdout:Invalid unit name "ceph-rbd-mirror@*.service" escaped as "ceph-rbd-mirror@\x2a.service". 2026-02-22T22:32:41.673 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:32:41.673 INFO:teuthology.orchestra.run.trial194.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-rbd-mirror.target → /usr/lib/systemd/system/ceph-rbd-mirror.target. 2026-02-22T22:32:41.673 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:41.775 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:50 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:41.859 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:41.859 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:41.859 INFO:teuthology.orchestra.run.trial198.stdout: Package Architecture Version Repository Size 2026-02-22T22:32:41.859 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:41.859 INFO:teuthology.orchestra.run.trial198.stdout:Installing: 2026-02-22T22:32:41.859 INFO:teuthology.orchestra.run.trial198.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T22:32:41.859 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:41.860 INFO:teuthology.orchestra.run.trial198.stdout:Transaction Summary 2026-02-22T22:32:41.860 INFO:teuthology.orchestra.run.trial198.stdout:================================================================================ 2026-02-22T22:32:41.860 INFO:teuthology.orchestra.run.trial198.stdout:Install 1 Package 2026-02-22T22:32:41.860 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:41.860 INFO:teuthology.orchestra.run.trial198.stdout:Total download size: 170 k 2026-02-22T22:32:41.860 INFO:teuthology.orchestra.run.trial198.stdout:Installed size: 490 k 2026-02-22T22:32:41.860 INFO:teuthology.orchestra.run.trial198.stdout:Downloading Packages: 2026-02-22T22:32:41.956 INFO:teuthology.orchestra.run.trial198.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.7 MB/s | 170 kB 00:00 2026-02-22T22:32:41.957 INFO:teuthology.orchestra.run.trial198.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:41.957 INFO:teuthology.orchestra.run.trial198.stdout:Total 1.7 MB/s | 170 kB 00:00 2026-02-22T22:32:41.958 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction check 2026-02-22T22:32:41.961 INFO:teuthology.orchestra.run.trial198.stdout:Transaction check succeeded. 2026-02-22T22:32:41.961 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction test 2026-02-22T22:32:41.992 INFO:teuthology.orchestra.run.trial198.stdout:Transaction test succeeded. 2026-02-22T22:32:41.992 INFO:teuthology.orchestra.run.trial198.stdout:Running transaction 2026-02-22T22:32:41.998 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:41.998 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:41.998 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:41.998 INFO:teuthology.orchestra.run.trial194.stdout: rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:41.998 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:41.998 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:42.026 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install rbd-nbd 2026-02-22T22:32:42.048 INFO:teuthology.orchestra.run.trial198.stdout: Preparing : 1/1 2026-02-22T22:32:42.065 INFO:teuthology.orchestra.run.trial198.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:42.150 INFO:teuthology.orchestra.run.trial198.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:42.209 INFO:teuthology.orchestra.run.trial198.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:42.209 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:42.209 INFO:teuthology.orchestra.run.trial198.stdout:Installed: 2026-02-22T22:32:42.209 INFO:teuthology.orchestra.run.trial198.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:42.209 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:42.209 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:42.238 DEBUG:teuthology.orchestra.run.trial198:> sudo yum -y install rbd-mirror 2026-02-22T22:32:42.341 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:51 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:42.426 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:42.426 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:42.426 INFO:teuthology.orchestra.run.trial194.stdout: Package Architecture Version Repository Size 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout:Installing: 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout: rbd-nbd x86_64 2:19.2.3-689.gf90f4eae.el9 ceph 170 k 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout:Transaction Summary 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout:================================================================================ 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout:Install 1 Package 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout:Total download size: 170 k 2026-02-22T22:32:42.427 INFO:teuthology.orchestra.run.trial194.stdout:Installed size: 490 k 2026-02-22T22:32:42.428 INFO:teuthology.orchestra.run.trial194.stdout:Downloading Packages: 2026-02-22T22:32:42.514 INFO:teuthology.orchestra.run.trial194.stdout:rbd-nbd-19.2.3-689.gf90f4eae.el9.x86_64.rpm 1.9 MB/s | 170 kB 00:00 2026-02-22T22:32:42.514 INFO:teuthology.orchestra.run.trial194.stdout:-------------------------------------------------------------------------------- 2026-02-22T22:32:42.514 INFO:teuthology.orchestra.run.trial194.stdout:Total 1.9 MB/s | 170 kB 00:00 2026-02-22T22:32:42.515 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction check 2026-02-22T22:32:42.518 INFO:teuthology.orchestra.run.trial194.stdout:Transaction check succeeded. 2026-02-22T22:32:42.518 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction test 2026-02-22T22:32:42.549 INFO:teuthology.orchestra.run.trial194.stdout:Transaction test succeeded. 2026-02-22T22:32:42.549 INFO:teuthology.orchestra.run.trial194.stdout:Running transaction 2026-02-22T22:32:42.558 INFO:teuthology.orchestra.run.trial198.stdout:Last metadata expiration check: 0:00:51 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:42.605 INFO:teuthology.orchestra.run.trial194.stdout: Preparing : 1/1 2026-02-22T22:32:42.621 INFO:teuthology.orchestra.run.trial194.stdout: Installing : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:42.623 INFO:teuthology.orchestra.run.trial198.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:42.642 INFO:teuthology.orchestra.run.trial198.stdout:Dependencies resolved. 2026-02-22T22:32:42.642 INFO:teuthology.orchestra.run.trial198.stdout:Nothing to do. 2026-02-22T22:32:42.642 INFO:teuthology.orchestra.run.trial198.stdout:Complete! 2026-02-22T22:32:42.667 DEBUG:teuthology.parallel:result is None 2026-02-22T22:32:42.710 INFO:teuthology.orchestra.run.trial194.stdout: Running scriptlet: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:42.768 INFO:teuthology.orchestra.run.trial194.stdout: Verifying : rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 1/1 2026-02-22T22:32:42.768 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:42.768 INFO:teuthology.orchestra.run.trial194.stdout:Installed: 2026-02-22T22:32:42.768 INFO:teuthology.orchestra.run.trial194.stdout: rbd-nbd-2:19.2.3-689.gf90f4eae.el9.x86_64 2026-02-22T22:32:42.768 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:42.768 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:42.798 DEBUG:teuthology.orchestra.run.trial194:> sudo yum -y install rbd-mirror 2026-02-22T22:32:43.113 INFO:teuthology.orchestra.run.trial194.stdout:Last metadata expiration check: 0:00:52 ago on Sun 22 Feb 2026 10:31:51 PM UTC. 2026-02-22T22:32:43.178 INFO:teuthology.orchestra.run.trial194.stdout:Package rbd-mirror-2:19.2.3-689.gf90f4eae.el9.x86_64 is already installed. 2026-02-22T22:32:43.196 INFO:teuthology.orchestra.run.trial194.stdout:Dependencies resolved. 2026-02-22T22:32:43.197 INFO:teuthology.orchestra.run.trial194.stdout:Nothing to do. 2026-02-22T22:32:43.197 INFO:teuthology.orchestra.run.trial194.stdout:Complete! 2026-02-22T22:32:43.225 DEBUG:teuthology.parallel:result is None 2026-02-22T22:32:43.225 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:32:43.386 DEBUG:teuthology.orchestra.run.trial194:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T22:32:43.407 INFO:teuthology.orchestra.run.trial194.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T22:32:43.407 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T22:32:43.407 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T22:32:43.409 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F9%2Fx86_64&sha1=f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:32:43.550 DEBUG:teuthology.orchestra.run.trial198:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2026-02-22T22:32:43.572 INFO:teuthology.orchestra.run.trial198.stdout:19.2.3-689.gf90f4eae.el9 2026-02-22T22:32:43.572 INFO:teuthology.packaging:The installed version of ceph is 19.2.3-689.gf90f4eae.el9 2026-02-22T22:32:43.572 INFO:teuthology.task.install:The correct ceph version 19.2.3-689.gf90f4eae is installed. 2026-02-22T22:32:43.575 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2026-02-22T22:32:43.575 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:43.575 DEBUG:teuthology.orchestra.run.trial194:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T22:32:43.602 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:32:43.602 DEBUG:teuthology.orchestra.run.trial198:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2026-02-22T22:32:43.636 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2026-02-22T22:32:43.636 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:43.636 DEBUG:teuthology.orchestra.run.trial194:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T22:32:43.666 DEBUG:teuthology.orchestra.run.trial194:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T22:32:43.731 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:32:43.731 DEBUG:teuthology.orchestra.run.trial198:> sudo dd of=/usr/bin/daemon-helper 2026-02-22T22:32:43.758 DEBUG:teuthology.orchestra.run.trial198:> sudo chmod a=rx -- /usr/bin/daemon-helper 2026-02-22T22:32:43.820 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2026-02-22T22:32:43.820 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:43.820 DEBUG:teuthology.orchestra.run.trial194:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T22:32:43.846 DEBUG:teuthology.orchestra.run.trial194:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T22:32:43.909 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:32:43.909 DEBUG:teuthology.orchestra.run.trial198:> sudo dd of=/usr/bin/adjust-ulimits 2026-02-22T22:32:43.934 DEBUG:teuthology.orchestra.run.trial198:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2026-02-22T22:32:43.999 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2026-02-22T22:32:43.999 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:43.999 DEBUG:teuthology.orchestra.run.trial194:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T22:32:44.025 DEBUG:teuthology.orchestra.run.trial194:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T22:32:44.087 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:32:44.087 DEBUG:teuthology.orchestra.run.trial198:> sudo dd of=/usr/bin/stdin-killer 2026-02-22T22:32:44.112 DEBUG:teuthology.orchestra.run.trial198:> sudo chmod a=rx -- /usr/bin/stdin-killer 2026-02-22T22:32:44.175 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T22:32:44.252 INFO:tasks.ceph:Making ceph log dir writeable by non-root... 2026-02-22T22:32:44.253 DEBUG:teuthology.orchestra.run.trial194:> sudo chmod 777 /var/log/ceph 2026-02-22T22:32:44.255 DEBUG:teuthology.orchestra.run.trial198:> sudo chmod 777 /var/log/ceph 2026-02-22T22:32:44.279 INFO:tasks.ceph:Disabling ceph logrotate... 2026-02-22T22:32:44.279 DEBUG:teuthology.orchestra.run.trial194:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T22:32:44.319 DEBUG:teuthology.orchestra.run.trial198:> sudo rm -f -- /etc/logrotate.d/ceph 2026-02-22T22:32:44.342 INFO:tasks.ceph:Creating extra log directories... 2026-02-22T22:32:44.342 DEBUG:teuthology.orchestra.run.trial194:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T22:32:44.383 DEBUG:teuthology.orchestra.run.trial198:> sudo install -d -m0777 -- /var/log/ceph/valgrind /var/log/ceph/profiling-logger 2026-02-22T22:32:44.423 INFO:tasks.ceph:Creating ceph cluster cluster1... 2026-02-22T22:32:44.423 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster1', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T22:32:44.423 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-low-osd-mem-target supported-random-distro$/{centos_latest} workloads/rbd-mirror-journal-bootstrap-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64242', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror_bootstrap.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial194.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOOwTOfNyp/XTybSmuH0uElIAmBOUI32nWYnTuSOjBlkNOFDVz3wDXzW+QYrxRFsxWd5I5/XoImMeyylNgM7qyE=', 'trial198.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEMJdPpeURyplAGCiXMMqIaEeQc0+Iz1U+I5KwvjiAzM2sOh2fsBPnQ35awYfBsrwVyu/68IPJQpYqM+5LHcQBY='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_bootstrap.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '1', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T22:32:44.424 DEBUG:teuthology.orchestra.run.trial194:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T22:32:44.448 DEBUG:teuthology.orchestra.run.trial198:> install -d -m0755 -- /home/ubuntu/cephtest/cluster1.data 2026-02-22T22:32:44.465 DEBUG:teuthology.orchestra.run.trial194:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:32:44.506 DEBUG:teuthology.orchestra.run.trial198:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:32:44.530 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:44.530 DEBUG:teuthology.orchestra.run.trial194:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T22:32:44.583 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T22:32:44.584 DEBUG:teuthology.orchestra.run.trial194:> stat /dev/vg_nvme/lv_1 2026-02-22T22:32:44.639 INFO:teuthology.orchestra.run.trial194.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T22:32:44.640 INFO:teuthology.orchestra.run.trial194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:32:44.640 INFO:teuthology.orchestra.run.trial194.stdout:Device: 6h/6d Inode: 1355 Links: 1 2026-02-22T22:32:44.640 INFO:teuthology.orchestra.run.trial194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:32:44.640 INFO:teuthology.orchestra.run.trial194.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:32:44.640 INFO:teuthology.orchestra.run.trial194.stdout:Access: 2026-02-22 22:32:41.817102692 +0000 2026-02-22T22:32:44.640 INFO:teuthology.orchestra.run.trial194.stdout:Modify: 2026-02-22 22:31:56.455946439 +0000 2026-02-22T22:32:44.640 INFO:teuthology.orchestra.run.trial194.stdout:Change: 2026-02-22 22:31:56.455946439 +0000 2026-02-22T22:32:44.640 INFO:teuthology.orchestra.run.trial194.stdout: Birth: 2026-02-22 22:31:56.455946439 +0000 2026-02-22T22:32:44.640 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T22:32:44.702 INFO:teuthology.orchestra.run.trial194.stderr:1+0 records in 2026-02-22T22:32:44.703 INFO:teuthology.orchestra.run.trial194.stderr:1+0 records out 2026-02-22T22:32:44.703 INFO:teuthology.orchestra.run.trial194.stderr:512 bytes copied, 0.000179012 s, 2.9 MB/s 2026-02-22T22:32:44.704 DEBUG:teuthology.orchestra.run.trial194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T22:32:44.762 DEBUG:teuthology.orchestra.run.trial194:> stat /dev/vg_nvme/lv_2 2026-02-22T22:32:44.818 INFO:teuthology.orchestra.run.trial194.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T22:32:44.818 INFO:teuthology.orchestra.run.trial194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:32:44.818 INFO:teuthology.orchestra.run.trial194.stdout:Device: 6h/6d Inode: 1338 Links: 1 2026-02-22T22:32:44.818 INFO:teuthology.orchestra.run.trial194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:32:44.819 INFO:teuthology.orchestra.run.trial194.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:32:44.819 INFO:teuthology.orchestra.run.trial194.stdout:Access: 2026-02-22 22:32:41.817102692 +0000 2026-02-22T22:32:44.819 INFO:teuthology.orchestra.run.trial194.stdout:Modify: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:44.819 INFO:teuthology.orchestra.run.trial194.stdout:Change: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:44.819 INFO:teuthology.orchestra.run.trial194.stdout: Birth: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:44.819 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T22:32:44.881 INFO:teuthology.orchestra.run.trial194.stderr:1+0 records in 2026-02-22T22:32:44.881 INFO:teuthology.orchestra.run.trial194.stderr:1+0 records out 2026-02-22T22:32:44.881 INFO:teuthology.orchestra.run.trial194.stderr:512 bytes copied, 0.000132957 s, 3.9 MB/s 2026-02-22T22:32:44.883 DEBUG:teuthology.orchestra.run.trial194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T22:32:44.940 DEBUG:teuthology.orchestra.run.trial194:> stat /dev/vg_nvme/lv_3 2026-02-22T22:32:44.996 INFO:teuthology.orchestra.run.trial194.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T22:32:44.996 INFO:teuthology.orchestra.run.trial194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:32:44.997 INFO:teuthology.orchestra.run.trial194.stdout:Device: 6h/6d Inode: 1335 Links: 1 2026-02-22T22:32:44.997 INFO:teuthology.orchestra.run.trial194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:32:44.997 INFO:teuthology.orchestra.run.trial194.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:32:44.997 INFO:teuthology.orchestra.run.trial194.stdout:Access: 2026-02-22 22:32:41.817102692 +0000 2026-02-22T22:32:44.997 INFO:teuthology.orchestra.run.trial194.stdout:Modify: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:44.997 INFO:teuthology.orchestra.run.trial194.stdout:Change: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:44.997 INFO:teuthology.orchestra.run.trial194.stdout: Birth: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:44.998 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T22:32:45.060 INFO:teuthology.orchestra.run.trial194.stderr:1+0 records in 2026-02-22T22:32:45.061 INFO:teuthology.orchestra.run.trial194.stderr:1+0 records out 2026-02-22T22:32:45.061 INFO:teuthology.orchestra.run.trial194.stderr:512 bytes copied, 0.00015093 s, 3.4 MB/s 2026-02-22T22:32:45.062 DEBUG:teuthology.orchestra.run.trial194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T22:32:45.121 DEBUG:teuthology.orchestra.run.trial194:> stat /dev/vg_nvme/lv_4 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout:Device: 6h/6d Inode: 1336 Links: 1 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout:Access: 2026-02-22 22:32:41.818102673 +0000 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout:Modify: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout:Change: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:45.178 INFO:teuthology.orchestra.run.trial194.stdout: Birth: 2026-02-22 22:31:56.454946457 +0000 2026-02-22T22:32:45.178 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T22:32:45.241 INFO:teuthology.orchestra.run.trial194.stderr:1+0 records in 2026-02-22T22:32:45.241 INFO:teuthology.orchestra.run.trial194.stderr:1+0 records out 2026-02-22T22:32:45.241 INFO:teuthology.orchestra.run.trial194.stderr:512 bytes copied, 0.00015596 s, 3.3 MB/s 2026-02-22T22:32:45.242 DEBUG:teuthology.orchestra.run.trial194:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T22:32:45.298 INFO:tasks.ceph:osd dev map: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:32:45.299 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial194.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:32:45.299 INFO:tasks.ceph:Generating config... 2026-02-22T22:32:45.299 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T22:32:45.299 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T22:32:45.299 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T22:32:45.299 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T22:32:45.299 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.299 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.300 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.300 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T22:32:45.300 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T22:32:45.300 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T22:32:45.300 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T22:32:45.300 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T22:32:45.300 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T22:32:45.300 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T22:32:45.300 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] osd memory target = 1610612736 2026-02-22T22:32:45.300 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T22:32:45.300 INFO:tasks.ceph:Setting up cluster1.mon.a... 2026-02-22T22:32:45.301 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster1.keyring 2026-02-22T22:32:45.373 INFO:teuthology.orchestra.run.trial194.stdout:creating /etc/ceph/cluster1.keyring 2026-02-22T22:32:45.376 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster1.keyring 2026-02-22T22:32:45.450 DEBUG:teuthology.orchestra.run.trial194:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:32:45.515 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster1.mon.a', '10.20.193.194')] 2026-02-22T22:32:45.516 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.194', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T22:32:45.518 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:45.518 DEBUG:teuthology.orchestra.run.trial194:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:32:45.574 DEBUG:teuthology.orchestra.run.trial194:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.194 --print /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:32:45.644 INFO:teuthology.orchestra.run.trial194.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:32:45.644 INFO:teuthology.orchestra.run.trial194.stdout:monmaptool: generated fsid 53304493-37aa-4143-9c19-51b919f55963 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:setting min_mon_release = quincy 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:epoch 0 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:fsid 53304493-37aa-4143-9c19-51b919f55963 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:last_changed 2026-02-22T22:32:45.643991+0000 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:created 2026-02-22T22:32:45.643991+0000 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:min_mon_release 17 (quincy) 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:election_strategy: 1 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:0: [v2:10.20.193.194:3300/0,v1:10.20.193.194:6789/0] mon.a 2026-02-22T22:32:45.645 INFO:teuthology.orchestra.run.trial194.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster1.monmap (1 monitors) 2026-02-22T22:32:45.647 DEBUG:teuthology.orchestra.run.trial194:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:32:45.703 INFO:tasks.ceph:Writing /etc/ceph/cluster1.conf for FSID 53304493-37aa-4143-9c19-51b919f55963... 2026-02-22T22:32:45.705 DEBUG:teuthology.orchestra.run.trial194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T22:32:45.746 DEBUG:teuthology.orchestra.run.trial198:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster1.conf && sudo chmod 0644 /etc/ceph/cluster1.conf > /dev/null 2026-02-22T22:32:45.782 INFO:teuthology.orchestra.run.trial194.stdout:[global] 2026-02-22T22:32:45.782 INFO:teuthology.orchestra.run.trial194.stdout: chdir = "" 2026-02-22T22:32:45.782 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:32:45.782 INFO:teuthology.orchestra.run.trial194.stdout: auth supported = cephx 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: filestore xattr use omap = true 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: mon clock drift allowed = 1.000 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: osd crush chooseleaf type = 0 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: auth debug = true 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: ms die on old message = true 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: ms die on bug = true 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.783 INFO:teuthology.orchestra.run.trial194.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: mon pg warn max object skew = 0 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: osd pool default size = 2 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: mon osd allow primary affinity = true 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: mon osd allow pg remap = true 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:32:45.784 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on no sortbitwise = false 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on too few osds = false 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: mon_allow_pool_size_one = true 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: osd default data pool replay window = 5 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: mon allow pool delete = true 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.785 INFO:teuthology.orchestra.run.trial194.stdout: mon cluster log file level = debug 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: debug asserts on shutdown = true 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: mon health detail to clog = false 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: mon host = 10.20.193.194 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: mon client directed command retry = 5 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on pool no app = False 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: ms inject socket failures = 5000 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: fsid = 53304493-37aa-4143-9c19-51b919f55963 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout:[osd] 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: osd journal size = 100 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.786 INFO:teuthology.orchestra.run.trial194.stdout: osd scrub load threshold = 5.0 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd scrub max interval = 600 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd mclock skip benchmark = true 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd recover clone overlap = true 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd recovery max chunk = 1048576 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd debug shutdown = true 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd debug op order = true 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd debug verify stray on activate = true 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: osd debug trim objects = true 2026-02-22T22:32:45.787 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: osd open classes on start = true 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: osd debug pg log writeout = true 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: osd map max advance = 10 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: journal zero on create = true 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: filestore apply finisher threads = 3 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.788 INFO:teuthology.orchestra.run.trial194.stdout: bdev debug aio = true 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: osd debug misdirected ops = true 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: bdev async discard = True 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: bdev enable discard = True 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: bluestore block size = 96636764160 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: bluestore fsck on mount = True 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: debug bluefs = 1/20 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: debug bluestore = 1/20 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: debug ms = 1 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: debug osd = 20 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: debug rocksdb = 4/10 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: mon osd full ratio = 0.9 2026-02-22T22:32:45.789 INFO:teuthology.orchestra.run.trial194.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: osd memory target = 1610612736 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: osd objectstore = bluestore 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout:[mgr] 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: debug ms = 1 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: debug mgr = 20 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: debug mon = 20 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: debug auth = 20 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:32:45.790 INFO:teuthology.orchestra.run.trial194.stdout: mgr/telemetry/nag = false 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout:[mon] 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: debug ms = 1 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: debug mon = 20 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: debug paxos = 20 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: debug auth = 20 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: mon data avail warn = 5 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: mon mgr mkfs grace = 240 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: mon osd reporter subtree level = osd 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: mon osd prime pg temp = true 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:32:45.791 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: # 1m isn't quite enough 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.792 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout:[client] 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: rgw cache enabled = true 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: rgw enable ops log = true 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: rgw enable usage log = true 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: debug rbd = 20 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: debug rbd_mirror = 15 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: log to stderr = False 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout: rbd default features = 125 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout:[mon.a] 2026-02-22T22:32:45.793 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.0] 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.1] 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.2] 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.3] 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.4] 2026-02-22T22:32:45.794 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.795 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.795 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.5] 2026-02-22T22:32:45.795 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.795 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.795 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.6] 2026-02-22T22:32:45.795 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.795 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.796 INFO:teuthology.orchestra.run.trial198.stdout:[global] 2026-02-22T22:32:45.796 INFO:teuthology.orchestra.run.trial198.stdout: chdir = "" 2026-02-22T22:32:45.796 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:32:45.796 INFO:teuthology.orchestra.run.trial198.stdout: auth supported = cephx 2026-02-22T22:32:45.796 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.796 INFO:teuthology.orchestra.run.trial198.stdout: filestore xattr use omap = true 2026-02-22T22:32:45.796 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.796 INFO:teuthology.orchestra.run.trial198.stdout: mon clock drift allowed = 1.000 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: osd crush chooseleaf type = 0 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: auth debug = true 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: ms die on old message = true 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: ms die on bug = true 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: mon pg warn max object skew = 0 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:32:45.797 INFO:teuthology.orchestra.run.trial198.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: osd pool default size = 2 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon osd allow primary affinity = true 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon osd allow pg remap = true 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on no sortbitwise = false 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on too few osds = false 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:32:45.798 INFO:teuthology.orchestra.run.trial198.stdout: mon_allow_pool_size_one = true 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: osd default data pool replay window = 5 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: mon allow pool delete = true 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: mon cluster log file level = debug 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: debug asserts on shutdown = true 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: mon health detail to clog = false 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: mon host = 10.20.193.194 2026-02-22T22:32:45.799 INFO:teuthology.orchestra.run.trial198.stdout: mon client directed command retry = 5 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on pool no app = False 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: ms inject socket failures = 5000 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: fsid = 53304493-37aa-4143-9c19-51b919f55963 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout:[osd] 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: osd journal size = 100 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: osd scrub load threshold = 5.0 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: osd scrub max interval = 600 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: osd mclock skip benchmark = true 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.800 INFO:teuthology.orchestra.run.trial198.stdout: osd recover clone overlap = true 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd recovery max chunk = 1048576 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd debug shutdown = true 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd debug op order = true 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd debug verify stray on activate = true 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd debug trim objects = true 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd open classes on start = true 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd debug pg log writeout = true 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.801 INFO:teuthology.orchestra.run.trial198.stdout: osd map max advance = 10 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: journal zero on create = true 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: filestore apply finisher threads = 3 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: bdev debug aio = true 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: osd debug misdirected ops = true 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: bdev async discard = True 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: bdev enable discard = True 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: bluestore block size = 96636764160 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: bluestore fsck on mount = True 2026-02-22T22:32:45.802 INFO:teuthology.orchestra.run.trial198.stdout: debug bluefs = 1/20 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: debug bluestore = 1/20 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: debug ms = 1 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: debug osd = 20 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: debug rocksdb = 4/10 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: mon osd full ratio = 0.9 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: osd memory target = 1610612736 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: osd objectstore = bluestore 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout:[mgr] 2026-02-22T22:32:45.803 INFO:teuthology.orchestra.run.trial198.stdout: debug ms = 1 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: debug mgr = 20 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: debug mon = 20 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: debug auth = 20 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: mgr/telemetry/nag = false 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout:[mon] 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: debug ms = 1 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: debug mon = 20 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: debug paxos = 20 2026-02-22T22:32:45.804 INFO:teuthology.orchestra.run.trial198.stdout: debug auth = 20 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: mon data avail warn = 5 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: mon mgr mkfs grace = 240 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: mon osd reporter subtree level = osd 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: mon osd prime pg temp = true 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:32:45.805 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: # 1m isn't quite enough 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout:[client] 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: rgw cache enabled = true 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: rgw enable ops log = true 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: rgw enable usage log = true 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:32:45.806 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout: debug rbd = 20 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout: debug rbd_mirror = 15 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout: log to stderr = False 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout: rbd default features = 125 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout:[mon.a] 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.0] 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.1] 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.807 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.2] 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.3] 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.4] 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.5] 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.6] 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:32:45.808 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:32:45.810 INFO:tasks.ceph:Creating admin key on cluster1.mon.a... 2026-02-22T22:32:45.810 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster1.keyring 2026-02-22T22:32:45.892 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T22:32:45.893 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:45.893 DEBUG:teuthology.orchestra.run.trial194:> dd if=/etc/ceph/cluster1.keyring of=/dev/stdout 2026-02-22T22:32:45.951 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:45.951 DEBUG:teuthology.orchestra.run.trial194:> dd if=/home/ubuntu/cephtest/cluster1.monmap of=/dev/stdout 2026-02-22T22:32:46.006 INFO:tasks.ceph:Sending monmap to node ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:32:46.007 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:46.007 DEBUG:teuthology.orchestra.run.trial194:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T22:32:46.007 DEBUG:teuthology.orchestra.run.trial194:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:32:46.080 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:32:46.080 DEBUG:teuthology.orchestra.run.trial194:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:32:46.136 INFO:tasks.ceph:Sending monmap to node ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:32:46.136 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:32:46.136 DEBUG:teuthology.orchestra.run.trial198:> sudo dd of=/etc/ceph/cluster1.keyring 2026-02-22T22:32:46.136 DEBUG:teuthology.orchestra.run.trial198:> sudo chmod 0644 /etc/ceph/cluster1.keyring 2026-02-22T22:32:46.170 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:32:46.170 DEBUG:teuthology.orchestra.run.trial198:> dd of=/home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:32:46.225 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T22:32:46.225 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T22:32:46.225 DEBUG:teuthology.orchestra.run.trial194:> sudo mkdir -p /var/lib/ceph/mgr/cluster1-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T22:32:46.269 INFO:teuthology.orchestra.run.trial194.stdout:creating /var/lib/ceph/mgr/cluster1-x/keyring 2026-02-22T22:32:46.272 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T22:32:46.272 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T22:32:46.272 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster1.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.0.keyring 2026-02-22T22:32:46.307 INFO:teuthology.orchestra.run.trial194.stdout:creating /etc/ceph/cluster1.client.0.keyring 2026-02-22T22:32:46.318 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster1.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T22:32:46.353 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster1.client.mirror.keyring 2026-02-22T22:32:46.364 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster1.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T22:32:46.438 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster1.client.mirror.0.keyring 2026-02-22T22:32:46.450 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster1.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T22:32:46.523 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster1.client.mirror.1.keyring 2026-02-22T22:32:46.534 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster1.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T22:32:46.608 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster1.client.mirror.2.keyring 2026-02-22T22:32:46.618 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster1.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T22:32:46.694 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster1.client.mirror.3.keyring 2026-02-22T22:32:46.705 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster1.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T22:32:46.779 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster1.client.mirror.4.keyring 2026-02-22T22:32:46.790 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster1.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T22:32:46.862 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster1.client.mirror.5.keyring 2026-02-22T22:32:46.872 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster1.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T22:32:46.944 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster1.client.mirror.6.keyring 2026-02-22T22:32:46.955 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T22:32:46.955 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial194.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:32:46.955 DEBUG:teuthology.orchestra.run.trial194:> sudo mkdir -p /var/lib/ceph/osd/cluster1-0 2026-02-22T22:32:46.981 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:32:46.981 INFO:tasks.ceph:role: cluster1.osd.0 2026-02-22T22:32:46.981 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:32:46.981 DEBUG:teuthology.orchestra.run.trial194:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T22:32:47.046 INFO:teuthology.orchestra.run.trial194.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:32:47.046 INFO:teuthology.orchestra.run.trial194.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:32:47.046 INFO:teuthology.orchestra.run.trial194.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:32:47.046 INFO:teuthology.orchestra.run.trial194.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:32:47.046 INFO:teuthology.orchestra.run.trial194.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:32:47.046 INFO:teuthology.orchestra.run.trial194.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:32:47.046 INFO:teuthology.orchestra.run.trial194.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:32:47.047 INFO:teuthology.orchestra.run.trial194.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:32:47.047 INFO:teuthology.orchestra.run.trial194.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:32:47.047 INFO:teuthology.orchestra.run.trial194.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:32:47.620 INFO:teuthology.orchestra.run.trial194.stdout:Discarding blocks...Done. 2026-02-22T22:32:47.622 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial194.front.sepia.ceph.com -o noatime 2026-02-22T22:32:47.623 DEBUG:teuthology.orchestra.run.trial194:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster1-0 2026-02-22T22:32:47.692 DEBUG:teuthology.orchestra.run.trial194:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-0 2026-02-22T22:32:47.760 DEBUG:teuthology.orchestra.run.trial194:> sudo mkdir -p /var/lib/ceph/osd/cluster1-1 2026-02-22T22:32:47.825 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:32:47.826 INFO:tasks.ceph:role: cluster1.osd.1 2026-02-22T22:32:47.826 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:32:47.826 DEBUG:teuthology.orchestra.run.trial194:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T22:32:47.892 INFO:teuthology.orchestra.run.trial194.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:32:47.893 INFO:teuthology.orchestra.run.trial194.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:32:48.452 INFO:teuthology.orchestra.run.trial194.stdout:Discarding blocks...Done. 2026-02-22T22:32:48.454 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial194.front.sepia.ceph.com -o noatime 2026-02-22T22:32:48.454 DEBUG:teuthology.orchestra.run.trial194:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster1-1 2026-02-22T22:32:48.522 DEBUG:teuthology.orchestra.run.trial194:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-1 2026-02-22T22:32:48.589 DEBUG:teuthology.orchestra.run.trial194:> sudo mkdir -p /var/lib/ceph/osd/cluster1-2 2026-02-22T22:32:48.653 INFO:tasks.ceph:roles_to_devs: {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:32:48.653 INFO:tasks.ceph:role: cluster1.osd.2 2026-02-22T22:32:48.653 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:32:48.653 DEBUG:teuthology.orchestra.run.trial194:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T22:32:48.717 INFO:teuthology.orchestra.run.trial194.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:32:48.718 INFO:teuthology.orchestra.run.trial194.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:32:49.276 INFO:teuthology.orchestra.run.trial194.stdout:Discarding blocks...Done. 2026-02-22T22:32:49.278 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial194.front.sepia.ceph.com -o noatime 2026-02-22T22:32:49.279 DEBUG:teuthology.orchestra.run.trial194:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster1-2 2026-02-22T22:32:49.347 DEBUG:teuthology.orchestra.run.trial194:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster1-2 2026-02-22T22:32:49.414 DEBUG:teuthology.orchestra.run.trial194:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:32:49.492 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:49.489+0000 7f5d3d20b780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-0/keyring: can't open /var/lib/ceph/osd/cluster1-0/keyring: (2) No such file or directory 2026-02-22T22:32:49.492 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:49.489+0000 7f5d3d20b780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-0/keyring 2026-02-22T22:32:49.492 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:49.489+0000 7f5d3d20b780 -1 bdev(0x560dd32ab800 /var/lib/ceph/osd/cluster1-0/block) open stat got: (1) Operation not permitted 2026-02-22T22:32:49.493 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:49.489+0000 7f5d3d20b780 -1 bluestore(/var/lib/ceph/osd/cluster1-0) _read_fsid unparsable uuid 2026-02-22T22:32:53.379 DEBUG:teuthology.orchestra.run.trial194:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-0 2026-02-22T22:32:53.446 DEBUG:teuthology.orchestra.run.trial194:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:32:53.524 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:53.521+0000 7ff3abf3d780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-1/keyring: can't open /var/lib/ceph/osd/cluster1-1/keyring: (2) No such file or directory 2026-02-22T22:32:53.524 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:53.521+0000 7ff3abf3d780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-1/keyring 2026-02-22T22:32:53.524 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:53.521+0000 7ff3abf3d780 -1 bdev(0x56109b563800 /var/lib/ceph/osd/cluster1-1/block) open stat got: (1) Operation not permitted 2026-02-22T22:32:53.524 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:53.522+0000 7ff3abf3d780 -1 bluestore(/var/lib/ceph/osd/cluster1-1) _read_fsid unparsable uuid 2026-02-22T22:32:57.415 DEBUG:teuthology.orchestra.run.trial194:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-1 2026-02-22T22:32:57.481 DEBUG:teuthology.orchestra.run.trial194:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster1 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:32:57.558 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:57.555+0000 7fade55b4780 -1 auth: error reading file: /var/lib/ceph/osd/cluster1-2/keyring: can't open /var/lib/ceph/osd/cluster1-2/keyring: (2) No such file or directory 2026-02-22T22:32:57.558 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:57.556+0000 7fade55b4780 -1 created new key in keyring /var/lib/ceph/osd/cluster1-2/keyring 2026-02-22T22:32:57.558 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:57.556+0000 7fade55b4780 -1 bdev(0x55e752b47800 /var/lib/ceph/osd/cluster1-2/block) open stat got: (1) Operation not permitted 2026-02-22T22:32:57.559 INFO:teuthology.orchestra.run.trial194.stderr:2026-02-22T22:32:57.556+0000 7fade55b4780 -1 bluestore(/var/lib/ceph/osd/cluster1-2) _read_fsid unparsable uuid 2026-02-22T22:33:01.445 DEBUG:teuthology.orchestra.run.trial194:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster1-2 2026-02-22T22:33:01.509 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T22:33:01.509 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:01.509 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/var/lib/ceph/mgr/cluster1-x/keyring of=/dev/stdout 2026-02-22T22:33:01.574 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:01.574 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/var/lib/ceph/osd/cluster1-0/keyring of=/dev/stdout 2026-02-22T22:33:01.638 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:01.638 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/var/lib/ceph/osd/cluster1-1/keyring of=/dev/stdout 2026-02-22T22:33:01.703 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:01.703 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/var/lib/ceph/osd/cluster1-2/keyring of=/dev/stdout 2026-02-22T22:33:01.766 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:01.767 DEBUG:teuthology.orchestra.run.trial194:> dd if=/etc/ceph/cluster1.client.0.keyring of=/dev/stdout 2026-02-22T22:33:01.823 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:01.823 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster1.client.mirror.keyring of=/dev/stdout 2026-02-22T22:33:01.842 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:01.842 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster1.client.mirror.0.keyring of=/dev/stdout 2026-02-22T22:33:01.898 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:01.899 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster1.client.mirror.1.keyring of=/dev/stdout 2026-02-22T22:33:01.956 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:01.957 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster1.client.mirror.2.keyring of=/dev/stdout 2026-02-22T22:33:02.014 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:02.014 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster1.client.mirror.3.keyring of=/dev/stdout 2026-02-22T22:33:02.072 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:02.072 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster1.client.mirror.4.keyring of=/dev/stdout 2026-02-22T22:33:02.130 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:02.130 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster1.client.mirror.5.keyring of=/dev/stdout 2026-02-22T22:33:02.186 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:02.187 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster1.client.mirror.6.keyring of=/dev/stdout 2026-02-22T22:33:02.243 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T22:33:02.244 DEBUG:teuthology.orchestra.run.trial194:> sudo tee -a /etc/ceph/cluster1.keyring 2026-02-22T22:33:02.268 INFO:teuthology.orchestra.run.trial194.stdout:[mgr.x] 2026-02-22T22:33:02.268 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpn58JEBAApfI9dBBw5zkNi1qP4q+DlQ== 2026-02-22T22:33:02.268 INFO:teuthology.orchestra.run.trial194.stdout:[osd.0] 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout: key = AQARhJtpmINGHRAA+sLToO5p8LIvDIi3xHC1eQ== 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout:[osd.1] 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAVhJtp6jcxHxAA2j43Hv9m2Acwz3uIMNQUXg== 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout:[osd.2] 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAZhJtp3R09IRAAKx0D2RMyGitAV62SIkPGDA== 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout:[client.0] 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpGtU+EhAAWVZA8PVMM+MaQdP/SwpA/Q== 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror] 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpB3MhFRAAE8tajg+W0JWSalBkWz8HVA== 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.0] 2026-02-22T22:33:02.269 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtp8twuGhAAsJEe9Q4YTv/fZoHG477dCg== 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.1] 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpl71JHxAAcpIhvWrYur3yWt73k4ckRg== 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.2] 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpC8pUJBAAuanczlqktczDGp+5Xpj7jw== 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.3] 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpZABwKRAAbGOK5dONCZf94eOwdV502g== 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.4] 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpp1qHLhAAg5eXoXk8YLDsf/D6gQJViQ== 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.5] 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpQ/h5MxAAojxvft2F7Ld6mpzcgRdyLg== 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.6] 2026-02-22T22:33:02.270 INFO:teuthology.orchestra.run.trial194.stdout: key = AQAOhJtpbcNiOBAASe4wt/GmX9IVa5DQKzskhA== 2026-02-22T22:33:02.272 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T22:33:02.353 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:33:02.435 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:33:02.517 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:33:02.596 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:02.678 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:02.759 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:02.841 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:02.923 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:03.003 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:03.086 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:03.167 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:03.249 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster1.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:03.331 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T22:33:03.331 DEBUG:teuthology.orchestra.run.trial194:> sudo mkdir -p /var/lib/ceph/mon/cluster1-a 2026-02-22T22:33:03.395 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster1 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster1.monmap --keyring /etc/ceph/cluster1.keyring 2026-02-22T22:33:03.482 DEBUG:teuthology.orchestra.run.trial194:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster1-a 2026-02-22T22:33:03.507 DEBUG:teuthology.orchestra.run.trial194:> rm -- /home/ubuntu/cephtest/cluster1.monmap 2026-02-22T22:33:03.564 INFO:tasks.ceph:Starting mon daemons in cluster cluster1... 2026-02-22T22:33:03.565 INFO:tasks.ceph.cluster1.mon.a:Restarting daemon 2026-02-22T22:33:03.565 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster1 -i a 2026-02-22T22:33:03.607 INFO:tasks.ceph.cluster1.mon.a:Started 2026-02-22T22:33:03.607 INFO:tasks.ceph:Starting mgr daemons in cluster cluster1... 2026-02-22T22:33:03.607 INFO:tasks.ceph.cluster1.mgr.x:Restarting daemon 2026-02-22T22:33:03.607 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster1 -i x 2026-02-22T22:33:03.610 INFO:tasks.ceph.cluster1.mgr.x:Started 2026-02-22T22:33:03.610 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T22:33:03.610 DEBUG:teuthology.orchestra.run.trial194:> sudo ceph --cluster cluster1 osd crush tunables default 2026-02-22T22:33:03.714 INFO:teuthology.orchestra.run.trial194.stderr:adjusted tunables profile to default 2026-02-22T22:33:03.722 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T22:33:03.722 INFO:tasks.ceph:Starting osd daemons in cluster cluster1... 2026-02-22T22:33:03.723 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:03.723 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/var/lib/ceph/osd/cluster1-0/fsid of=/dev/stdout 2026-02-22T22:33:03.747 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:03.748 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/var/lib/ceph/osd/cluster1-1/fsid of=/dev/stdout 2026-02-22T22:33:03.812 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:03.812 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/var/lib/ceph/osd/cluster1-2/fsid of=/dev/stdout 2026-02-22T22:33:03.877 DEBUG:teuthology.orchestra.run.trial194:> sudo ceph --cluster cluster1 osd new 952bf1aa-0b8e-4788-b6ee-4090a0547472 0 2026-02-22T22:33:04.022 INFO:teuthology.orchestra.run.trial194.stdout:0 2026-02-22T22:33:04.030 DEBUG:teuthology.orchestra.run.trial194:> sudo ceph --cluster cluster1 osd new 8bc82af6-3bff-4b7b-96e2-53757ee66e72 1 2026-02-22T22:33:04.139 INFO:teuthology.orchestra.run.trial194.stdout:1 2026-02-22T22:33:04.148 DEBUG:teuthology.orchestra.run.trial194:> sudo ceph --cluster cluster1 osd new ec376c34-9c21-490d-a010-6ee8ac90b04d 2 2026-02-22T22:33:04.221 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.218+0000 7fb9125a5140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T22:33:04.255 INFO:teuthology.orchestra.run.trial194.stdout:2 2026-02-22T22:33:04.264 INFO:tasks.ceph.cluster1.osd.0:Restarting daemon 2026-02-22T22:33:04.264 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 0 2026-02-22T22:33:04.266 INFO:tasks.ceph.cluster1.osd.0:Started 2026-02-22T22:33:04.266 INFO:tasks.ceph.cluster1.osd.1:Restarting daemon 2026-02-22T22:33:04.267 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 1 2026-02-22T22:33:04.269 INFO:tasks.ceph.cluster1.osd.1:Started 2026-02-22T22:33:04.269 INFO:tasks.ceph.cluster1.osd.2:Restarting daemon 2026-02-22T22:33:04.269 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster1 -i 2 2026-02-22T22:33:04.271 INFO:tasks.ceph.cluster1.osd.2:Started 2026-02-22T22:33:04.272 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:33:04.346 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.344+0000 7fb9125a5140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T22:33:04.374 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.371+0000 7fb9125a5140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T22:33:04.374 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:04.375 INFO:teuthology.orchestra.run.trial194.stdout:{"epoch":5,"fsid":"53304493-37aa-4143-9c19-51b919f55963","created":"2026-02-22T22:33:03.668145+0000","modified":"2026-02-22T22:33:04.254047+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:33:04.254047+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"952bf1aa-0b8e-4788-b6ee-4090a0547472","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8bc82af6-3bff-4b7b-96e2-53757ee66e72","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"ec376c34-9c21-490d-a010-6ee8ac90b04d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:04.383 INFO:tasks.ceph.ceph_manager.cluster1:[] 2026-02-22T22:33:04.383 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T22:33:04.408 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.405+0000 7fb9125a5140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T22:33:04.431 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.428+0000 7fb9125a5140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T22:33:04.457 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.454+0000 7fb9125a5140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T22:33:04.512 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.510+0000 7fb9125a5140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T22:33:04.573 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T22:33:04.574 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T22:33:04.574 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr: from numpy import show_config as show_numpy_config 2026-02-22T22:33:04.577 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.574+0000 7fb9125a5140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T22:33:04.602 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.600+0000 7fb9125a5140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T22:33:04.658 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.655+0000 7fb9125a5140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T22:33:04.686 DEBUG:teuthology.orchestra.run.trial194:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:33:04.701 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.698+0000 7fb9125a5140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T22:33:04.727 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.724+0000 7fb9125a5140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T22:33:04.752 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.749+0000 7fb9125a5140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T22:33:04.777 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.775+0000 7fb9125a5140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T22:33:04.778 INFO:teuthology.misc.health.trial194.stdout: 2026-02-22T22:33:04.778 INFO:teuthology.misc.health.trial194.stdout:{"epoch":5,"fsid":"53304493-37aa-4143-9c19-51b919f55963","created":"2026-02-22T22:33:03.668145+0000","modified":"2026-02-22T22:33:04.254047+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:33:04.254047+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"952bf1aa-0b8e-4788-b6ee-4090a0547472","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8bc82af6-3bff-4b7b-96e2-53757ee66e72","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"ec376c34-9c21-490d-a010-6ee8ac90b04d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:04.785 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:33:04.800 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.798+0000 7fb9125a5140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T22:33:04.994 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:04.992+0000 7fb9125a5140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T22:33:05.048 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.047+0000 7fb9125a5140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T22:33:05.070 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.069+0000 7fb9125a5140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T22:33:05.094 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.091+0000 7fb9125a5140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T22:33:05.147 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.145+0000 7fb9125a5140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:33:05.167 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.166+0000 7fb9125a5140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T22:33:05.221 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.218+0000 7fb9125a5140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:33:05.621 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.619+0000 7fb9125a5140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T22:33:05.643 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.641+0000 7fb9125a5140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T22:33:05.981 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:05.979+0000 7fb9125a5140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T22:33:06.203 INFO:tasks.ceph.cluster1.osd.1.trial194.stderr:2026-02-22T22:33:06.201+0000 7fe17181a780 -1 Falling back to public interface 2026-02-22T22:33:06.204 INFO:tasks.ceph.cluster1.osd.0.trial194.stderr:2026-02-22T22:33:06.201+0000 7f4b19618780 -1 Falling back to public interface 2026-02-22T22:33:06.204 INFO:tasks.ceph.cluster1.osd.1.trial194.stderr:2026-02-22T22:33:06.201+0000 7fe170214640 -1 PosixStack listen unable to listen on v2:0.0.0.0:6802/0: (98) Address already in use 2026-02-22T22:33:06.212 INFO:tasks.ceph.cluster1.osd.2.trial194.stderr:2026-02-22T22:33:06.210+0000 7f3695611780 -1 Falling back to public interface 2026-02-22T22:33:06.988 INFO:tasks.ceph.cluster1.mgr.x.trial194.stderr:2026-02-22T22:33:06.986+0000 7fb82fd26640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T22:33:10.707 INFO:tasks.ceph.cluster1.osd.1.trial194.stderr:2026-02-22T22:33:10.705+0000 7fe17181a780 -1 osd.1 0 log_to_monitors true 2026-02-22T22:33:10.715 INFO:tasks.ceph.cluster1.osd.0.trial194.stderr:2026-02-22T22:33:10.713+0000 7f4b19618780 -1 osd.0 0 log_to_monitors true 2026-02-22T22:33:10.748 INFO:tasks.ceph.cluster1.osd.2.trial194.stderr:2026-02-22T22:33:10.747+0000 7f3695611780 -1 osd.2 0 log_to_monitors true 2026-02-22T22:33:11.088 DEBUG:teuthology.orchestra.run.trial194:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:33:11.241 INFO:teuthology.misc.health.trial194.stdout: 2026-02-22T22:33:11.242 INFO:teuthology.misc.health.trial194.stdout:{"epoch":6,"fsid":"53304493-37aa-4143-9c19-51b919f55963","created":"2026-02-22T22:33:03.668145+0000","modified":"2026-02-22T22:33:10.758625+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:33:04.254047+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"952bf1aa-0b8e-4788-b6ee-4090a0547472","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8bc82af6-3bff-4b7b-96e2-53757ee66e72","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"ec376c34-9c21-490d-a010-6ee8ac90b04d","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:11.249 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:33:11.775 INFO:tasks.ceph.cluster1.osd.1.trial194.stderr:2026-02-22T22:33:11.774+0000 7fe16d1da640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T22:33:11.776 INFO:tasks.ceph.cluster1.osd.0.trial194.stderr:2026-02-22T22:33:11.775+0000 7f4b14fda640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T22:33:11.776 INFO:tasks.ceph.cluster1.osd.2.trial194.stderr:2026-02-22T22:33:11.775+0000 7f3691017640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T22:33:11.777 INFO:tasks.ceph.cluster1.osd.1.trial194.stderr:2026-02-22T22:33:11.776+0000 7fe16500d640 -1 osd.1 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:33:11.778 INFO:tasks.ceph.cluster1.osd.0.trial194.stderr:2026-02-22T22:33:11.776+0000 7f4b0ce0d640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:33:11.778 INFO:tasks.ceph.cluster1.osd.2.trial194.stderr:2026-02-22T22:33:11.776+0000 7f3688e4a640 -1 osd.2 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:33:17.551 DEBUG:teuthology.orchestra.run.trial194:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:33:17.703 INFO:teuthology.misc.health.trial194.stdout: 2026-02-22T22:33:17.703 INFO:teuthology.misc.health.trial194.stdout:{"epoch":12,"fsid":"53304493-37aa-4143-9c19-51b919f55963","created":"2026-02-22T22:33:03.668145+0000","modified":"2026-02-22T22:33:16.768342+0000","last_up_change":"2026-02-22T22:33:12.761432+0000","last_in_change":"2026-02-22T22:33:04.254047+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:33:13.995981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"952bf1aa-0b8e-4788-b6ee-4090a0547472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6802","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6804","nonce":286964534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6806","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6808","nonce":286964534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6814","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6816","nonce":286964534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6810","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6812","nonce":286964534}]},"public_addr":"10.20.193.194:6804/286964534","cluster_addr":"10.20.193.194:6808/286964534","heartbeat_back_addr":"10.20.193.194:6816/286964534","heartbeat_front_addr":"10.20.193.194:6812/286964534","state":["exists","up"]},{"osd":1,"uuid":"8bc82af6-3bff-4b7b-96e2-53757ee66e72","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":10,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6803","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6805","nonce":3688961049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6807","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6809","nonce":3688961049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6815","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6817","nonce":3688961049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6811","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6813","nonce":3688961049}]},"public_addr":"10.20.193.194:6805/3688961049","cluster_addr":"10.20.193.194:6809/3688961049","heartbeat_back_addr":"10.20.193.194:6817/3688961049","heartbeat_front_addr":"10.20.193.194:6813/3688961049","state":["exists","up"]},{"osd":2,"uuid":"ec376c34-9c21-490d-a010-6ee8ac90b04d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6818","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6819","nonce":1852489993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6820","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6821","nonce":1852489993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6824","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6825","nonce":1852489993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6822","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6823","nonce":1852489993}]},"public_addr":"10.20.193.194:6819/1852489993","cluster_addr":"10.20.193.194:6821/1852489993","heartbeat_back_addr":"10.20.193.194:6825/1852489993","heartbeat_front_addr":"10.20.193.194:6823/1852489993","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.706930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.729631+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.737583+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:17.710 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T22:33:17.710 INFO:tasks.ceph:Creating RBD pool 2026-02-22T22:33:17.710 DEBUG:teuthology.orchestra.run.trial194:> sudo ceph --cluster cluster1 osd pool create rbd 8 2026-02-22T22:33:18.774 INFO:teuthology.orchestra.run.trial194.stderr:pool 'rbd' created 2026-02-22T22:33:18.785 DEBUG:teuthology.orchestra.run.trial194:> rbd --cluster cluster1 pool init rbd 2026-02-22T22:33:21.787 INFO:tasks.ceph:Starting mds daemons in cluster cluster1... 2026-02-22T22:33:21.787 INFO:tasks.ceph:Waiting until cluster1 daemons up and pgs clean... 2026-02-22T22:33:21.788 INFO:tasks.ceph.ceph_manager.cluster1:waiting for mgr available 2026-02-22T22:33:21.788 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 mgr dump --format=json 2026-02-22T22:33:21.788 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T22:33:21.970 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:21.977 INFO:teuthology.orchestra.run.trial194.stdout:{"epoch":5,"flags":0,"active_gid":4102,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6800","nonce":2052669915},{"type":"v1","addr":"10.20.193.194:6801","nonce":2052669915}]},"active_addr":"10.20.193.194:6801/2052669915","active_change":"2026-02-22T22:33:05.981264+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.194:0","nonce":3392429493}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.194:0","nonce":160621602}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.194:0","nonce":399085008}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.194:0","nonce":406950422}]}]} 2026-02-22T22:33:21.981 INFO:tasks.ceph.ceph_manager.cluster1:mgr available! 2026-02-22T22:33:21.981 INFO:tasks.ceph.ceph_manager.cluster1:waiting for all up 2026-02-22T22:33:21.981 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:33:22.140 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:22.140 INFO:teuthology.orchestra.run.trial194.stdout:{"epoch":16,"fsid":"53304493-37aa-4143-9c19-51b919f55963","created":"2026-02-22T22:33:03.668145+0000","modified":"2026-02-22T22:33:21.777803+0000","last_up_change":"2026-02-22T22:33:12.761432+0000","last_in_change":"2026-02-22T22:33:04.254047+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:33:13.995981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:33:17.872309+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"952bf1aa-0b8e-4788-b6ee-4090a0547472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6802","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6804","nonce":286964534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6806","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6808","nonce":286964534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6814","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6816","nonce":286964534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6810","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6812","nonce":286964534}]},"public_addr":"10.20.193.194:6804/286964534","cluster_addr":"10.20.193.194:6808/286964534","heartbeat_back_addr":"10.20.193.194:6816/286964534","heartbeat_front_addr":"10.20.193.194:6812/286964534","state":["exists","up"]},{"osd":1,"uuid":"8bc82af6-3bff-4b7b-96e2-53757ee66e72","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6803","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6805","nonce":3688961049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6807","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6809","nonce":3688961049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6815","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6817","nonce":3688961049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6811","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6813","nonce":3688961049}]},"public_addr":"10.20.193.194:6805/3688961049","cluster_addr":"10.20.193.194:6809/3688961049","heartbeat_back_addr":"10.20.193.194:6817/3688961049","heartbeat_front_addr":"10.20.193.194:6813/3688961049","state":["exists","up"]},{"osd":2,"uuid":"ec376c34-9c21-490d-a010-6ee8ac90b04d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6818","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6819","nonce":1852489993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6820","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6821","nonce":1852489993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6824","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6825","nonce":1852489993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6822","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6823","nonce":1852489993}]},"public_addr":"10.20.193.194:6819/1852489993","cluster_addr":"10.20.193.194:6821/1852489993","heartbeat_back_addr":"10.20.193.194:6825/1852489993","heartbeat_front_addr":"10.20.193.194:6823/1852489993","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.706930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.729631+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.737583+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:22.149 INFO:tasks.ceph.ceph_manager.cluster1:all up! 2026-02-22T22:33:22.149 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:33:22.308 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:22.309 INFO:teuthology.orchestra.run.trial194.stdout:{"epoch":16,"fsid":"53304493-37aa-4143-9c19-51b919f55963","created":"2026-02-22T22:33:03.668145+0000","modified":"2026-02-22T22:33:21.777803+0000","last_up_change":"2026-02-22T22:33:12.761432+0000","last_in_change":"2026-02-22T22:33:04.254047+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:33:13.995981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:33:17.872309+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"952bf1aa-0b8e-4788-b6ee-4090a0547472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6802","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6804","nonce":286964534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6806","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6808","nonce":286964534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6814","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6816","nonce":286964534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6810","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6812","nonce":286964534}]},"public_addr":"10.20.193.194:6804/286964534","cluster_addr":"10.20.193.194:6808/286964534","heartbeat_back_addr":"10.20.193.194:6816/286964534","heartbeat_front_addr":"10.20.193.194:6812/286964534","state":["exists","up"]},{"osd":1,"uuid":"8bc82af6-3bff-4b7b-96e2-53757ee66e72","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6803","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6805","nonce":3688961049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6807","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6809","nonce":3688961049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6815","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6817","nonce":3688961049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6811","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6813","nonce":3688961049}]},"public_addr":"10.20.193.194:6805/3688961049","cluster_addr":"10.20.193.194:6809/3688961049","heartbeat_back_addr":"10.20.193.194:6817/3688961049","heartbeat_front_addr":"10.20.193.194:6813/3688961049","state":["exists","up"]},{"osd":2,"uuid":"ec376c34-9c21-490d-a010-6ee8ac90b04d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":13,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6818","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6819","nonce":1852489993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6820","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6821","nonce":1852489993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6824","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6825","nonce":1852489993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6822","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6823","nonce":1852489993}]},"public_addr":"10.20.193.194:6819/1852489993","cluster_addr":"10.20.193.194:6821/1852489993","heartbeat_back_addr":"10.20.193.194:6825/1852489993","heartbeat_front_addr":"10.20.193.194:6823/1852489993","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.706930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.729631+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.737583+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:22.317 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 flush_pg_stats 2026-02-22T22:33:22.317 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 flush_pg_stats 2026-02-22T22:33:22.317 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 flush_pg_stats 2026-02-22T22:33:22.403 INFO:teuthology.orchestra.run.trial194.stdout:34359738371 2026-02-22T22:33:22.403 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:33:22.404 INFO:teuthology.orchestra.run.trial194.stdout:34359738371 2026-02-22T22:33:22.405 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:33:22.406 INFO:teuthology.orchestra.run.trial194.stdout:34359738372 2026-02-22T22:33:22.406 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:33:22.607 INFO:teuthology.orchestra.run.trial194.stdout:34359738370 2026-02-22T22:33:22.608 INFO:teuthology.orchestra.run.trial194.stdout:34359738371 2026-02-22T22:33:22.613 INFO:teuthology.orchestra.run.trial194.stdout:34359738370 2026-02-22T22:33:22.616 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.0 2026-02-22T22:33:22.617 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.1 2026-02-22T22:33:22.622 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.2 2026-02-22T22:33:23.617 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:33:23.618 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:33:23.623 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:33:23.781 INFO:teuthology.orchestra.run.trial194.stdout:34359738370 2026-02-22T22:33:23.784 INFO:teuthology.orchestra.run.trial194.stdout:34359738371 2026-02-22T22:33:23.792 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.0 2026-02-22T22:33:23.794 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738371 for osd.1 2026-02-22T22:33:23.820 INFO:teuthology.orchestra.run.trial194.stdout:34359738370 2026-02-22T22:33:23.829 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738370 for osd.2 2026-02-22T22:33:24.794 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.0 2026-02-22T22:33:24.795 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.1 2026-02-22T22:33:24.830 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd last-stat-seq osd.2 2026-02-22T22:33:24.956 INFO:teuthology.orchestra.run.trial194.stdout:34359738372 2026-02-22T22:33:24.959 INFO:teuthology.orchestra.run.trial194.stdout:34359738373 2026-02-22T22:33:24.966 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738372 for osd.0 2026-02-22T22:33:24.967 DEBUG:teuthology.parallel:result is None 2026-02-22T22:33:24.969 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738372 got 34359738373 for osd.1 2026-02-22T22:33:24.970 DEBUG:teuthology.parallel:result is None 2026-02-22T22:33:25.001 INFO:teuthology.orchestra.run.trial194.stdout:34359738372 2026-02-22T22:33:25.010 INFO:tasks.ceph.ceph_manager.cluster1:need seq 34359738371 got 34359738372 for osd.2 2026-02-22T22:33:25.011 DEBUG:teuthology.parallel:result is None 2026-02-22T22:33:25.011 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:33:25.011 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:33:25.211 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:25.211 INFO:teuthology.orchestra.run.trial194.stderr:dumped all 2026-02-22T22:33:25.219 INFO:teuthology.orchestra.run.trial194.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:33:23.985279+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":11,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82488,"kb_used_data":1576,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033032,"statfs":{"total":289910292480,"available":289825824768,"internally_reserved":0,"allocated":1613824,"data_stored":1303096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.211253"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783670+0000","last_change":"2026-02-22T22:33:21.783670+0000","last_active":"2026-02-22T22:33:21.783670+0000","last_peered":"2026-02-22T22:33:21.783670+0000","last_clean":"2026-02-22T22:33:21.783670+0000","last_became_active":"2026-02-22T22:33:19.779975+0000","last_became_peered":"2026-02-22T22:33:19.779975+0000","last_unstale":"2026-02-22T22:33:21.783670+0000","last_undegraded":"2026-02-22T22:33:21.783670+0000","last_fullsized":"2026-02-22T22:33:21.783670+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:41:03.479104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085832499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783880+0000","last_change":"2026-02-22T22:33:21.783880+0000","last_active":"2026-02-22T22:33:21.783880+0000","last_peered":"2026-02-22T22:33:21.783880+0000","last_clean":"2026-02-22T22:33:21.783880+0000","last_became_active":"2026-02-22T22:33:19.779948+0000","last_became_peered":"2026-02-22T22:33:19.779948+0000","last_unstale":"2026-02-22T22:33:21.783880+0000","last_undegraded":"2026-02-22T22:33:21.783880+0000","last_fullsized":"2026-02-22T22:33:21.783880+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:43.595634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001084284,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783616+0000","last_change":"2026-02-22T22:33:21.783616+0000","last_active":"2026-02-22T22:33:21.783616+0000","last_peered":"2026-02-22T22:33:21.783616+0000","last_clean":"2026-02-22T22:33:21.783616+0000","last_became_active":"2026-02-22T22:33:19.779809+0000","last_became_peered":"2026-02-22T22:33:19.779809+0000","last_unstale":"2026-02-22T22:33:21.783616+0000","last_undegraded":"2026-02-22T22:33:21.783616+0000","last_fullsized":"2026-02-22T22:33:21.783616+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:07.737454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078483800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783394+0000","last_change":"2026-02-22T22:33:21.783394+0000","last_active":"2026-02-22T22:33:21.783394+0000","last_peered":"2026-02-22T22:33:21.783394+0000","last_clean":"2026-02-22T22:33:21.783394+0000","last_became_active":"2026-02-22T22:33:19.779840+0000","last_became_peered":"2026-02-22T22:33:19.779840+0000","last_unstale":"2026-02-22T22:33:21.783394+0000","last_undegraded":"2026-02-22T22:33:21.783394+0000","last_fullsized":"2026-02-22T22:33:21.783394+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:39:30.893086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000614622,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.782077+0000","last_change":"2026-02-22T22:33:21.781441+0000","last_active":"2026-02-22T22:33:21.782077+0000","last_peered":"2026-02-22T22:33:21.782077+0000","last_clean":"2026-02-22T22:33:21.782077+0000","last_became_active":"2026-02-22T22:33:19.780216+0000","last_became_peered":"2026-02-22T22:33:19.780216+0000","last_unstale":"2026-02-22T22:33:21.782077+0000","last_undegraded":"2026-02-22T22:33:21.782077+0000","last_fullsized":"2026-02-22T22:33:21.782077+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:18:55.136618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049561099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.780155+0000","last_change":"2026-02-22T22:33:21.780155+0000","last_active":"2026-02-22T22:33:21.780155+0000","last_peered":"2026-02-22T22:33:21.780155+0000","last_clean":"2026-02-22T22:33:21.780155+0000","last_became_active":"2026-02-22T22:33:19.780151+0000","last_became_peered":"2026-02-22T22:33:19.780151+0000","last_unstale":"2026-02-22T22:33:21.780155+0000","last_undegraded":"2026-02-22T22:33:21.780155+0000","last_fullsized":"2026-02-22T22:33:21.780155+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:58:34.778892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029422600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.780152+0000","last_change":"2026-02-22T22:33:21.780152+0000","last_active":"2026-02-22T22:33:21.780152+0000","last_peered":"2026-02-22T22:33:21.780152+0000","last_clean":"2026-02-22T22:33:21.780152+0000","last_became_active":"2026-02-22T22:33:19.780160+0000","last_became_peered":"2026-02-22T22:33:19.780160+0000","last_unstale":"2026-02-22T22:33:21.780152+0000","last_undegraded":"2026-02-22T22:33:21.780152+0000","last_fullsized":"2026-02-22T22:33:21.780152+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:25:02.825325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034237599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783221+0000","last_change":"2026-02-22T22:33:21.783221+0000","last_active":"2026-02-22T22:33:21.783221+0000","last_peered":"2026-02-22T22:33:21.783221+0000","last_clean":"2026-02-22T22:33:21.783221+0000","last_became_active":"2026-02-22T22:33:19.779118+0000","last_became_peered":"2026-02-22T22:33:19.779118+0000","last_unstale":"2026-02-22T22:33:21.783221+0000","last_undegraded":"2026-02-22T22:33:21.783221+0000","last_fullsized":"2026-02-22T22:33:21.783221+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:07:18.496977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049514999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.782856+0000","last_change":"2026-02-22T22:33:15.817593+0000","last_active":"2026-02-22T22:33:21.782856+0000","last_peered":"2026-02-22T22:33:21.782856+0000","last_clean":"2026-02-22T22:33:21.782856+0000","last_became_active":"2026-02-22T22:33:15.817125+0000","last_became_peered":"2026-02-22T22:33:15.817125+0000","last_unstale":"2026-02-22T22:33:21.782856+0000","last_undegraded":"2026-02-22T22:33:21.782856+0000","last_fullsized":"2026-02-22T22:33:21.782856+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_clean_scrub_stamp":"2026-02-22T22:33:14.765836+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:24:54.631757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27104,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344736,"statfs":{"total":96636764160,"available":96609009664,"internally_reserved":0,"allocated":139264,"data_stored":40774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":631161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":631161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:33:25.221 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:33:25.380 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:25.380 INFO:teuthology.orchestra.run.trial194.stderr:dumped all 2026-02-22T22:33:25.388 INFO:teuthology.orchestra.run.trial194.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2026-02-22T22:33:23.985279+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":11,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82488,"kb_used_data":1576,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033032,"statfs":{"total":289910292480,"available":289825824768,"internally_reserved":0,"allocated":1613824,"data_stored":1303096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.211253"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783670+0000","last_change":"2026-02-22T22:33:21.783670+0000","last_active":"2026-02-22T22:33:21.783670+0000","last_peered":"2026-02-22T22:33:21.783670+0000","last_clean":"2026-02-22T22:33:21.783670+0000","last_became_active":"2026-02-22T22:33:19.779975+0000","last_became_peered":"2026-02-22T22:33:19.779975+0000","last_unstale":"2026-02-22T22:33:21.783670+0000","last_undegraded":"2026-02-22T22:33:21.783670+0000","last_fullsized":"2026-02-22T22:33:21.783670+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:41:03.479104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085832499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783880+0000","last_change":"2026-02-22T22:33:21.783880+0000","last_active":"2026-02-22T22:33:21.783880+0000","last_peered":"2026-02-22T22:33:21.783880+0000","last_clean":"2026-02-22T22:33:21.783880+0000","last_became_active":"2026-02-22T22:33:19.779948+0000","last_became_peered":"2026-02-22T22:33:19.779948+0000","last_unstale":"2026-02-22T22:33:21.783880+0000","last_undegraded":"2026-02-22T22:33:21.783880+0000","last_fullsized":"2026-02-22T22:33:21.783880+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:43.595634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001084284,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783616+0000","last_change":"2026-02-22T22:33:21.783616+0000","last_active":"2026-02-22T22:33:21.783616+0000","last_peered":"2026-02-22T22:33:21.783616+0000","last_clean":"2026-02-22T22:33:21.783616+0000","last_became_active":"2026-02-22T22:33:19.779809+0000","last_became_peered":"2026-02-22T22:33:19.779809+0000","last_unstale":"2026-02-22T22:33:21.783616+0000","last_undegraded":"2026-02-22T22:33:21.783616+0000","last_fullsized":"2026-02-22T22:33:21.783616+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:07.737454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078483800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783394+0000","last_change":"2026-02-22T22:33:21.783394+0000","last_active":"2026-02-22T22:33:21.783394+0000","last_peered":"2026-02-22T22:33:21.783394+0000","last_clean":"2026-02-22T22:33:21.783394+0000","last_became_active":"2026-02-22T22:33:19.779840+0000","last_became_peered":"2026-02-22T22:33:19.779840+0000","last_unstale":"2026-02-22T22:33:21.783394+0000","last_undegraded":"2026-02-22T22:33:21.783394+0000","last_fullsized":"2026-02-22T22:33:21.783394+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:39:30.893086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000614622,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"16'2","reported_seq":21,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.782077+0000","last_change":"2026-02-22T22:33:21.781441+0000","last_active":"2026-02-22T22:33:21.782077+0000","last_peered":"2026-02-22T22:33:21.782077+0000","last_clean":"2026-02-22T22:33:21.782077+0000","last_became_active":"2026-02-22T22:33:19.780216+0000","last_became_peered":"2026-02-22T22:33:19.780216+0000","last_unstale":"2026-02-22T22:33:21.782077+0000","last_undegraded":"2026-02-22T22:33:21.782077+0000","last_fullsized":"2026-02-22T22:33:21.782077+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:18:55.136618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049561099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.780155+0000","last_change":"2026-02-22T22:33:21.780155+0000","last_active":"2026-02-22T22:33:21.780155+0000","last_peered":"2026-02-22T22:33:21.780155+0000","last_clean":"2026-02-22T22:33:21.780155+0000","last_became_active":"2026-02-22T22:33:19.780151+0000","last_became_peered":"2026-02-22T22:33:19.780151+0000","last_unstale":"2026-02-22T22:33:21.780155+0000","last_undegraded":"2026-02-22T22:33:21.780155+0000","last_fullsized":"2026-02-22T22:33:21.780155+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:58:34.778892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029422600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.780152+0000","last_change":"2026-02-22T22:33:21.780152+0000","last_active":"2026-02-22T22:33:21.780152+0000","last_peered":"2026-02-22T22:33:21.780152+0000","last_clean":"2026-02-22T22:33:21.780152+0000","last_became_active":"2026-02-22T22:33:19.780160+0000","last_became_peered":"2026-02-22T22:33:19.780160+0000","last_unstale":"2026-02-22T22:33:21.780152+0000","last_undegraded":"2026-02-22T22:33:21.780152+0000","last_fullsized":"2026-02-22T22:33:21.780152+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:25:02.825325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034237599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"14'1","reported_seq":20,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.783221+0000","last_change":"2026-02-22T22:33:21.783221+0000","last_active":"2026-02-22T22:33:21.783221+0000","last_peered":"2026-02-22T22:33:21.783221+0000","last_clean":"2026-02-22T22:33:21.783221+0000","last_became_active":"2026-02-22T22:33:19.779118+0000","last_became_peered":"2026-02-22T22:33:19.779118+0000","last_unstale":"2026-02-22T22:33:21.783221+0000","last_undegraded":"2026-02-22T22:33:21.783221+0000","last_fullsized":"2026-02-22T22:33:21.783221+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:07:18.496977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049514999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"11'72","reported_seq":110,"reported_epoch":16,"state":"active+clean","last_fresh":"2026-02-22T22:33:21.782856+0000","last_change":"2026-02-22T22:33:15.817593+0000","last_active":"2026-02-22T22:33:21.782856+0000","last_peered":"2026-02-22T22:33:21.782856+0000","last_clean":"2026-02-22T22:33:21.782856+0000","last_became_active":"2026-02-22T22:33:15.817125+0000","last_became_peered":"2026-02-22T22:33:15.817125+0000","last_unstale":"2026-02-22T22:33:21.782856+0000","last_undegraded":"2026-02-22T22:33:21.782856+0000","last_fullsized":"2026-02-22T22:33:21.782856+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_clean_scrub_stamp":"2026-02-22T22:33:14.765836+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:24:54.631757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738372,"num_pgs":0,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27104,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344736,"statfs":{"total":96636764160,"available":96609009664,"internally_reserved":0,"allocated":139264,"data_stored":40774,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738373,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":631161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738372,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27692,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344148,"statfs":{"total":96636764160,"available":96608407552,"internally_reserved":0,"allocated":737280,"data_stored":631161,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:33:25.390 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:33:25.390 INFO:tasks.ceph:Waiting until ceph cluster cluster1 is healthy... 2026-02-22T22:33:25.390 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy 2026-02-22T22:33:25.390 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 health --format=json 2026-02-22T22:33:25.565 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:25.565 INFO:teuthology.orchestra.run.trial194.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T22:33:25.574 INFO:tasks.ceph.ceph_manager.cluster1:wait_until_healthy done 2026-02-22T22:33:25.574 INFO:teuthology.run_tasks:Running task ceph... 2026-02-22T22:33:25.584 INFO:tasks.ceph:Creating ceph cluster cluster2... 2026-02-22T22:33:25.585 INFO:tasks.ceph:config {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'fs': 'xfs', 'mkfs_options': None, 'mount_options': None, 'skip_mgr_daemons': False, 'log_ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'cpu_profile': set(), 'cluster': 'cluster2', 'mon_bind_msgr2': True, 'mon_bind_addrvec': True} 2026-02-22T22:33:25.585 INFO:tasks.ceph:ctx.config {'archive_path': '/home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242', 'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'description': 'rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-low-osd-mem-target supported-random-distro$/{centos_latest} workloads/rbd-mirror-journal-bootstrap-workunit}', 'email': 'yweinste@redhat.com', 'first_in_suite': False, 'flavor': 'default', 'job_id': '64242', 'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}, 'last_in_suite': False, 'machine_type': 'trial', 'meta': [{'desc': 'run two ceph clusters and install rbd-mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': 'configure the permissions for client.mirror'}, {'desc': '2 ceph clusters with 1 mon and 3 osds each'}, {'desc': 'run the rbd_mirror_bootstrap.sh workunit to test the rbd-mirror daemon'}], 'name': 'yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial', 'no_nested_subset': False, 'openstack': [{'volumes': {'count': 3, 'size': 30}}], 'os_type': 'centos', 'os_version': '9.stream', 'overrides': {'admin_socket': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid'}, 'ceph': {'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'selinux': {'allowlist': ['scontext=system_u:system_r:getty_t:s0']}, 'thrashosds': {'bdev_inject_crash': 2, 'bdev_inject_crash_probability': 0.5}, 'workunit': {'branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, 'owner': 'scheduled_yuriw@soko04.front.sepia.ceph.com', 'priority': 69, 'repo': 'https://github.com/ceph/ceph-ci.git', 'roles': [['cluster1.mon.a', 'cluster1.mgr.x', 'cluster2.mgr.x', 'cluster1.osd.0', 'cluster1.osd.1', 'cluster1.osd.2', 'cluster1.client.0', 'cluster2.client.0'], ['cluster2.mon.a', 'cluster2.osd.0', 'cluster2.osd.1', 'cluster2.osd.2', 'cluster1.client.mirror', 'cluster1.client.mirror.0', 'cluster1.client.mirror.1', 'cluster1.client.mirror.2', 'cluster1.client.mirror.3', 'cluster1.client.mirror.4', 'cluster1.client.mirror.5', 'cluster1.client.mirror.6', 'cluster2.client.mirror', 'cluster2.client.mirror.0', 'cluster2.client.mirror.1', 'cluster2.client.mirror.2', 'cluster2.client.mirror.3', 'cluster2.client.mirror.4', 'cluster2.client.mirror.5', 'cluster2.client.mirror.6']], 'seed': 9196, 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'sleep_before_teardown': 0, 'subset': '111/120000', 'suite': 'rbd', 'suite_branch': 'wip-yuri6-testing-2026-02-22-1629-squid', 'suite_path': '/home/teuthworker/src/github.com_ceph_ceph-c_f90f4eaebb28b619cf76e80cbbe997656da69579/qa', 'suite_relpath': 'qa', 'suite_repo': 'https://github.com/ceph/ceph-ci.git', 'suite_sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579', 'targets': {'trial194.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBOOwTOfNyp/XTybSmuH0uElIAmBOUI32nWYnTuSOjBlkNOFDVz3wDXzW+QYrxRFsxWd5I5/XoImMeyylNgM7qyE=', 'trial198.front.sepia.ceph.com': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEMJdPpeURyplAGCiXMMqIaEeQc0+Iz1U+I5KwvjiAzM2sOh2fsBPnQ35awYfBsrwVyu/68IPJQpYqM+5LHcQBY='}, 'tasks': [{'internal.check_packages': None}, {'internal.buildpackages_prep': None}, {'internal.save_config': None}, {'internal.check_lock': None}, {'internal.add_remotes': None}, {'console_log': None}, {'internal.connect': None}, {'internal.push_inventory': None}, {'internal.serialize_remote_roles': None}, {'internal.check_conflict': None}, {'internal.check_ceph_data': None}, {'internal.vm_setup': None}, {'kernel': {'branch': 'distro', 'kdb': True, 'sha1': 'distro'}}, {'internal.base': None}, {'internal.archive_upload': None}, {'internal.archive': None}, {'internal.coredump': None}, {'internal.sudo': None}, {'internal.syslog': None}, {'internal.timer': None}, {'pcp': None}, {'selinux': None}, {'ansible.cephlab': None}, {'clock': None}, {'install': {'extra_packages': ['rbd-mirror'], 'flavor': 'default', 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster1', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'ceph': {'cluster': 'cluster2', 'conf': {'client': {'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'global': {'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}}, 'flavor': 'default', 'fs': 'xfs', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)', '\\(OSD_SLOW_PING_TIME'], 'sha1': 'f90f4eaebb28b619cf76e80cbbe997656da69579'}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.4 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.5 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.6 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'exec': {'cluster1.client.mirror.0': ["sudo ceph --cluster cluster1 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster1 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror mon 'profile rbd-mirror-peer' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.0 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.1 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.2 mon 'profile rbd-mirror' osd 'profile rbd'", "sudo ceph --cluster cluster2 auth caps client.mirror.3 mon 'profile rbd-mirror' osd 'profile rbd'"]}}, {'workunit': {'clients': {'cluster1.client.mirror': ['rbd/rbd_mirror_bootstrap.sh']}, 'env': {'CEPH_ARGS': '', 'MIRROR_POOL_MODE': 'pool', 'RBD_MIRROR_INSTANCES': '1', 'RBD_MIRROR_MODE': 'journal', 'RBD_MIRROR_USE_EXISTING_CLUSTER': '1'}}}], 'teuthology': {'fragments_dropped': [], 'meta': {}, 'postmerge': []}, 'teuthology_branch': 'main', 'teuthology_sha1': 'fa17720d0088c3ac28e473468bfc79eeaff5cd38', 'timestamp': '2026-02-22_19:35:38', 'tube': 'trial', 'user': 'yuriw', 'verbose': True, 'worker_log': '/home/teuthworker/mnt/teuthology/worker_logs/dispatcher.trial.3318653'} 2026-02-22T22:33:25.586 DEBUG:teuthology.orchestra.run.trial194:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T22:33:25.588 DEBUG:teuthology.orchestra.run.trial198:> install -d -m0755 -- /home/ubuntu/cephtest/cluster2.data 2026-02-22T22:33:25.606 DEBUG:teuthology.orchestra.run.trial194:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:33:25.645 DEBUG:teuthology.orchestra.run.trial198:> sudo install -d -m0777 -- /var/run/ceph 2026-02-22T22:33:25.672 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:25.672 DEBUG:teuthology.orchestra.run.trial198:> dd if=/scratch_devs of=/dev/stdout 2026-02-22T22:33:25.730 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2026-02-22T22:33:25.730 DEBUG:teuthology.orchestra.run.trial198:> stat /dev/vg_nvme/lv_1 2026-02-22T22:33:25.787 INFO:teuthology.orchestra.run.trial198.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2026-02-22T22:33:25.787 INFO:teuthology.orchestra.run.trial198.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:33:25.787 INFO:teuthology.orchestra.run.trial198.stdout:Device: 6h/6d Inode: 1308 Links: 1 2026-02-22T22:33:25.787 INFO:teuthology.orchestra.run.trial198.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:33:25.787 INFO:teuthology.orchestra.run.trial198.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:33:25.787 INFO:teuthology.orchestra.run.trial198.stdout:Access: 2026-02-22 22:32:41.278080314 +0000 2026-02-22T22:33:25.787 INFO:teuthology.orchestra.run.trial198.stdout:Modify: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:25.788 INFO:teuthology.orchestra.run.trial198.stdout:Change: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:25.788 INFO:teuthology.orchestra.run.trial198.stdout: Birth: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:25.788 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2026-02-22T22:33:25.852 INFO:teuthology.orchestra.run.trial198.stderr:1+0 records in 2026-02-22T22:33:25.852 INFO:teuthology.orchestra.run.trial198.stderr:1+0 records out 2026-02-22T22:33:25.852 INFO:teuthology.orchestra.run.trial198.stderr:512 bytes copied, 0.000146452 s, 3.5 MB/s 2026-02-22T22:33:25.853 DEBUG:teuthology.orchestra.run.trial198:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2026-02-22T22:33:25.910 DEBUG:teuthology.orchestra.run.trial198:> stat /dev/vg_nvme/lv_2 2026-02-22T22:33:25.966 INFO:teuthology.orchestra.run.trial198.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2026-02-22T22:33:25.966 INFO:teuthology.orchestra.run.trial198.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:33:25.966 INFO:teuthology.orchestra.run.trial198.stdout:Device: 6h/6d Inode: 1312 Links: 1 2026-02-22T22:33:25.967 INFO:teuthology.orchestra.run.trial198.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:33:25.967 INFO:teuthology.orchestra.run.trial198.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:33:25.967 INFO:teuthology.orchestra.run.trial198.stdout:Access: 2026-02-22 22:32:41.279080305 +0000 2026-02-22T22:33:25.967 INFO:teuthology.orchestra.run.trial198.stdout:Modify: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:25.967 INFO:teuthology.orchestra.run.trial198.stdout:Change: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:25.967 INFO:teuthology.orchestra.run.trial198.stdout: Birth: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:25.967 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2026-02-22T22:33:26.029 INFO:teuthology.orchestra.run.trial198.stderr:1+0 records in 2026-02-22T22:33:26.029 INFO:teuthology.orchestra.run.trial198.stderr:1+0 records out 2026-02-22T22:33:26.030 INFO:teuthology.orchestra.run.trial198.stderr:512 bytes copied, 0.000151963 s, 3.4 MB/s 2026-02-22T22:33:26.031 DEBUG:teuthology.orchestra.run.trial198:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2026-02-22T22:33:26.089 DEBUG:teuthology.orchestra.run.trial198:> stat /dev/vg_nvme/lv_3 2026-02-22T22:33:26.146 INFO:teuthology.orchestra.run.trial198.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2026-02-22T22:33:26.146 INFO:teuthology.orchestra.run.trial198.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:33:26.146 INFO:teuthology.orchestra.run.trial198.stdout:Device: 6h/6d Inode: 1320 Links: 1 2026-02-22T22:33:26.146 INFO:teuthology.orchestra.run.trial198.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:33:26.147 INFO:teuthology.orchestra.run.trial198.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:33:26.147 INFO:teuthology.orchestra.run.trial198.stdout:Access: 2026-02-22 22:32:41.279080305 +0000 2026-02-22T22:33:26.147 INFO:teuthology.orchestra.run.trial198.stdout:Modify: 2026-02-22 22:31:57.899579338 +0000 2026-02-22T22:33:26.147 INFO:teuthology.orchestra.run.trial198.stdout:Change: 2026-02-22 22:31:57.899579338 +0000 2026-02-22T22:33:26.147 INFO:teuthology.orchestra.run.trial198.stdout: Birth: 2026-02-22 22:31:57.899579338 +0000 2026-02-22T22:33:26.147 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2026-02-22T22:33:26.211 INFO:teuthology.orchestra.run.trial198.stderr:1+0 records in 2026-02-22T22:33:26.211 INFO:teuthology.orchestra.run.trial198.stderr:1+0 records out 2026-02-22T22:33:26.211 INFO:teuthology.orchestra.run.trial198.stderr:512 bytes copied, 0.000231079 s, 2.2 MB/s 2026-02-22T22:33:26.213 DEBUG:teuthology.orchestra.run.trial198:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2026-02-22T22:33:26.272 DEBUG:teuthology.orchestra.run.trial198:> stat /dev/vg_nvme/lv_4 2026-02-22T22:33:26.328 INFO:teuthology.orchestra.run.trial198.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2026-02-22T22:33:26.329 INFO:teuthology.orchestra.run.trial198.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2026-02-22T22:33:26.329 INFO:teuthology.orchestra.run.trial198.stdout:Device: 6h/6d Inode: 1293 Links: 1 2026-02-22T22:33:26.329 INFO:teuthology.orchestra.run.trial198.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2026-02-22T22:33:26.329 INFO:teuthology.orchestra.run.trial198.stdout:Context: system_u:object_r:device_t:s0 2026-02-22T22:33:26.329 INFO:teuthology.orchestra.run.trial198.stdout:Access: 2026-02-22 22:32:41.279080305 +0000 2026-02-22T22:33:26.329 INFO:teuthology.orchestra.run.trial198.stdout:Modify: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:26.329 INFO:teuthology.orchestra.run.trial198.stdout:Change: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:26.330 INFO:teuthology.orchestra.run.trial198.stdout: Birth: 2026-02-22 22:31:57.898579350 +0000 2026-02-22T22:33:26.330 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2026-02-22T22:33:26.392 INFO:teuthology.orchestra.run.trial198.stderr:1+0 records in 2026-02-22T22:33:26.392 INFO:teuthology.orchestra.run.trial198.stderr:1+0 records out 2026-02-22T22:33:26.392 INFO:teuthology.orchestra.run.trial198.stderr:512 bytes copied, 0.000212776 s, 2.4 MB/s 2026-02-22T22:33:26.393 DEBUG:teuthology.orchestra.run.trial198:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2026-02-22T22:33:26.450 INFO:tasks.ceph:osd dev map: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:33:26.450 INFO:tasks.ceph:remote_to_roles_to_devs: {Remote(name='ubuntu@trial198.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:33:26.451 INFO:tasks.ceph:Generating config... 2026-02-22T22:33:26.453 INFO:tasks.ceph:[client] debug rbd = 20 2026-02-22T22:33:26.453 INFO:tasks.ceph:[client] debug rbd_mirror = 15 2026-02-22T22:33:26.453 INFO:tasks.ceph:[client] log to stderr = False 2026-02-22T22:33:26.453 INFO:tasks.ceph:[client] rbd default features = 125 2026-02-22T22:33:26.453 INFO:tasks.ceph:[client.mirror.0] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.453 INFO:tasks.ceph:[client.mirror.0] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.453 INFO:tasks.ceph:[client.mirror.1] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.453 INFO:tasks.ceph:[client.mirror.1] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.2] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.2] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.3] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.3] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.4] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.4] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.5] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.5] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.6] admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.454 INFO:tasks.ceph:[client.mirror.6] pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.454 INFO:tasks.ceph:[global] mon client directed command retry = 5 2026-02-22T22:33:26.455 INFO:tasks.ceph:[global] mon warn on pool no app = False 2026-02-22T22:33:26.455 INFO:tasks.ceph:[global] ms inject socket failures = 5000 2026-02-22T22:33:26.455 INFO:tasks.ceph:[mgr] debug mgr = 20 2026-02-22T22:33:26.455 INFO:tasks.ceph:[mgr] debug ms = 1 2026-02-22T22:33:26.455 INFO:tasks.ceph:[mon] debug mon = 20 2026-02-22T22:33:26.455 INFO:tasks.ceph:[mon] debug ms = 1 2026-02-22T22:33:26.455 INFO:tasks.ceph:[mon] debug paxos = 20 2026-02-22T22:33:26.455 INFO:tasks.ceph:[osd] bdev async discard = True 2026-02-22T22:33:26.455 INFO:tasks.ceph:[osd] bdev enable discard = True 2026-02-22T22:33:26.455 INFO:tasks.ceph:[osd] bluestore block size = 96636764160 2026-02-22T22:33:26.455 INFO:tasks.ceph:[osd] bluestore fsck on mount = True 2026-02-22T22:33:26.455 INFO:tasks.ceph:[osd] debug bluefs = 1/20 2026-02-22T22:33:26.456 INFO:tasks.ceph:[osd] debug bluestore = 1/20 2026-02-22T22:33:26.456 INFO:tasks.ceph:[osd] debug ms = 1 2026-02-22T22:33:26.456 INFO:tasks.ceph:[osd] debug osd = 20 2026-02-22T22:33:26.456 INFO:tasks.ceph:[osd] debug rocksdb = 4/10 2026-02-22T22:33:26.456 INFO:tasks.ceph:[osd] mon osd backfillfull_ratio = 0.85 2026-02-22T22:33:26.456 INFO:tasks.ceph:[osd] mon osd full ratio = 0.9 2026-02-22T22:33:26.456 INFO:tasks.ceph:[osd] mon osd nearfull ratio = 0.8 2026-02-22T22:33:26.456 INFO:tasks.ceph:[osd] osd failsafe full ratio = 0.95 2026-02-22T22:33:26.457 INFO:tasks.ceph:[osd] osd memory target = 1610612736 2026-02-22T22:33:26.457 INFO:tasks.ceph:[osd] osd objectstore = bluestore 2026-02-22T22:33:26.457 INFO:tasks.ceph:Setting up cluster2.mon.a... 2026-02-22T22:33:26.457 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring /etc/ceph/cluster2.keyring 2026-02-22T22:33:26.524 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.keyring 2026-02-22T22:33:26.527 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=mon. /etc/ceph/cluster2.keyring 2026-02-22T22:33:26.604 DEBUG:teuthology.orchestra.run.trial198:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:33:26.668 DEBUG:tasks.ceph:Ceph mon addresses: [('cluster2.mon.a', '10.20.193.198')] 2026-02-22T22:33:26.669 DEBUG:tasks.ceph:writing out conf {'global': {'chdir': '', 'pid file': '/var/run/ceph/$cluster-$name.pid', 'auth supported': 'cephx', 'filestore xattr use omap': 'true', 'mon clock drift allowed': '1.000', 'osd crush chooseleaf type': '0', 'auth debug': 'true', 'ms die on old message': 'true', 'ms die on bug': 'true', 'mon max pg per osd': '10000', 'mon pg warn max object skew': '0', 'osd_pool_default_pg_autoscale_mode': 'off', 'osd pool default size': '2', 'mon osd allow primary affinity': 'true', 'mon osd allow pg remap': 'true', 'mon warn on legacy crush tunables': 'false', 'mon warn on crush straw calc version zero': 'false', 'mon warn on no sortbitwise': 'false', 'mon warn on osd down out interval zero': 'false', 'mon warn on too few osds': 'false', 'mon_warn_on_pool_pg_num_not_power_of_two': 'false', 'mon_warn_on_pool_no_redundancy': 'false', 'mon_allow_pool_size_one': 'true', 'osd pool default erasure code profile': 'plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd', 'osd default data pool replay window': '5', 'mon allow pool delete': 'true', 'mon cluster log file level': 'debug', 'debug asserts on shutdown': 'true', 'mon health detail to clog': 'false', 'mon host': '10.20.193.198', 'mon client directed command retry': 5, 'mon warn on pool no app': False, 'ms inject socket failures': 5000}, 'osd': {'osd journal size': '100', 'osd scrub load threshold': '5.0', 'osd scrub max interval': '600', 'osd mclock profile': 'high_recovery_ops', 'osd mclock skip benchmark': 'true', 'osd recover clone overlap': 'true', 'osd recovery max chunk': '1048576', 'osd debug shutdown': 'true', 'osd debug op order': 'true', 'osd debug verify stray on activate': 'true', 'osd debug trim objects': 'true', 'osd open classes on start': 'true', 'osd debug pg log writeout': 'true', 'osd deep scrub update digest min age': '30', 'osd map max advance': '10', 'journal zero on create': 'true', 'filestore ondisk finisher threads': '3', 'filestore apply finisher threads': '3', 'bdev debug aio': 'true', 'osd debug misdirected ops': 'true', 'bdev async discard': True, 'bdev enable discard': True, 'bluestore block size': 96636764160, 'bluestore fsck on mount': True, 'debug bluefs': '1/20', 'debug bluestore': '1/20', 'debug ms': 1, 'debug osd': 20, 'debug rocksdb': '4/10', 'mon osd backfillfull_ratio': 0.85, 'mon osd full ratio': 0.9, 'mon osd nearfull ratio': 0.8, 'osd failsafe full ratio': 0.95, 'osd memory target': 1610612736, 'osd objectstore': 'bluestore'}, 'mgr': {'debug ms': 1, 'debug mgr': 20, 'debug mon': '20', 'debug auth': '20', 'mon reweight min pgs per osd': '4', 'mon reweight min bytes per osd': '10', 'mgr/telemetry/nag': 'false'}, 'mon': {'debug ms': 1, 'debug mon': 20, 'debug paxos': 20, 'debug auth': '20', 'mon data avail warn': '5', 'mon mgr mkfs grace': '240', 'mon reweight min pgs per osd': '4', 'mon osd reporter subtree level': 'osd', 'mon osd prime pg temp': 'true', 'mon reweight min bytes per osd': '10', 'auth mon ticket ttl': '660', 'auth service ticket ttl': '240', 'mon_warn_on_insecure_global_id_reclaim': 'false', 'mon_warn_on_insecure_global_id_reclaim_allowed': 'false', 'mon_down_mkfs_grace': '2m', 'mon_warn_on_filestore_osds': 'false'}, 'client': {'rgw cache enabled': 'true', 'rgw enable ops log': 'true', 'rgw enable usage log': 'true', 'log file': '/var/log/ceph/$cluster-$name.$pid.log', 'admin socket': '/var/run/ceph/$cluster-$name.$pid.asok', 'debug rbd': 20, 'debug rbd_mirror': 15, 'log to stderr': False, 'rbd default features': 125}, 'mon.a': {}, 'client.mirror.0': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.1': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.2': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.3': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.4': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.5': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}, 'client.mirror.6': {'admin socket': '/var/run/ceph/rbd-mirror.$cluster-$name.asok', 'pid file': '/var/run/ceph/rbd-mirror.$cluster-$name.pid'}} 2026-02-22T22:33:26.670 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:26.670 DEBUG:teuthology.orchestra.run.trial198:> dd of=/home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:33:26.727 DEBUG:teuthology.orchestra.run.trial198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage monmaptool -c /home/ubuntu/cephtest/ceph.tmp.conf --create --clobber --enable-all-features --add a 10.20.193.198 --print /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:33:26.798 INFO:teuthology.orchestra.run.trial198.stdout:monmaptool: monmap file /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:monmaptool: generated fsid 3163850b-09a4-4a3d-86d6-c3514600e52a 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:setting min_mon_release = quincy 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:epoch 0 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:fsid 3163850b-09a4-4a3d-86d6-c3514600e52a 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:last_changed 2026-02-22T22:33:26.799489+0000 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:created 2026-02-22T22:33:26.799489+0000 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:min_mon_release 17 (quincy) 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:election_strategy: 1 2026-02-22T22:33:26.799 INFO:teuthology.orchestra.run.trial198.stdout:0: [v2:10.20.193.198:3300/0,v1:10.20.193.198:6789/0] mon.a 2026-02-22T22:33:26.800 INFO:teuthology.orchestra.run.trial198.stdout:monmaptool: writing epoch 0 to /home/ubuntu/cephtest/cluster2.monmap (1 monitors) 2026-02-22T22:33:26.801 DEBUG:teuthology.orchestra.run.trial198:> rm -- /home/ubuntu/cephtest/ceph.tmp.conf 2026-02-22T22:33:26.857 INFO:tasks.ceph:Writing /etc/ceph/cluster2.conf for FSID 3163850b-09a4-4a3d-86d6-c3514600e52a... 2026-02-22T22:33:26.858 DEBUG:teuthology.orchestra.run.trial194:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T22:33:26.861 DEBUG:teuthology.orchestra.run.trial198:> sudo mkdir -p /etc/ceph && sudo chmod 0755 /etc/ceph && sudo tee /etc/ceph/cluster2.conf && sudo chmod 0644 /etc/ceph/cluster2.conf > /dev/null 2026-02-22T22:33:26.901 INFO:teuthology.orchestra.run.trial194.stdout:[global] 2026-02-22T22:33:26.901 INFO:teuthology.orchestra.run.trial194.stdout: chdir = "" 2026-02-22T22:33:26.901 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:33:26.901 INFO:teuthology.orchestra.run.trial194.stdout: auth supported = cephx 2026-02-22T22:33:26.901 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.901 INFO:teuthology.orchestra.run.trial194.stdout: filestore xattr use omap = true 2026-02-22T22:33:26.901 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: mon clock drift allowed = 1.000 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: osd crush chooseleaf type = 0 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: auth debug = true 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: ms die on old message = true 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: ms die on bug = true 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: mon pg warn max object skew = 0 2026-02-22T22:33:26.902 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: osd pool default size = 2 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: mon osd allow primary affinity = true 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: mon osd allow pg remap = true 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on no sortbitwise = false 2026-02-22T22:33:26.903 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on too few osds = false 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: mon_allow_pool_size_one = true 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: osd default data pool replay window = 5 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: mon allow pool delete = true 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.904 INFO:teuthology.orchestra.run.trial194.stdout: mon cluster log file level = debug 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: debug asserts on shutdown = true 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: mon health detail to clog = false 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: mon host = 10.20.193.198 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: mon client directed command retry = 5 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: mon warn on pool no app = False 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: ms inject socket failures = 5000 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: fsid = 3163850b-09a4-4a3d-86d6-c3514600e52a 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout:[osd] 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: osd journal size = 100 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.905 INFO:teuthology.orchestra.run.trial194.stdout: osd scrub load threshold = 5.0 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd scrub max interval = 600 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd mclock skip benchmark = true 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd recover clone overlap = true 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd recovery max chunk = 1048576 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd debug shutdown = true 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd debug op order = true 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd debug verify stray on activate = true 2026-02-22T22:33:26.906 INFO:teuthology.orchestra.run.trial194.stdout: osd debug trim objects = true 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: osd open classes on start = true 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: osd debug pg log writeout = true 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: osd map max advance = 10 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: journal zero on create = true 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:33:26.907 INFO:teuthology.orchestra.run.trial194.stdout: filestore apply finisher threads = 3 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: bdev debug aio = true 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: osd debug misdirected ops = true 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: bdev async discard = True 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: bdev enable discard = True 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: bluestore block size = 96636764160 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: bluestore fsck on mount = True 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: debug bluefs = 1/20 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: debug bluestore = 1/20 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: debug ms = 1 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: debug osd = 20 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: debug rocksdb = 4/10 2026-02-22T22:33:26.908 INFO:teuthology.orchestra.run.trial194.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: mon osd full ratio = 0.9 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: osd memory target = 1610612736 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: osd objectstore = bluestore 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout:[mgr] 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: debug ms = 1 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: debug mgr = 20 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: debug mon = 20 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: debug auth = 20 2026-02-22T22:33:26.909 INFO:teuthology.orchestra.run.trial194.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: mgr/telemetry/nag = false 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout:[mon] 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: debug ms = 1 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: debug mon = 20 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: debug paxos = 20 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: debug auth = 20 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: mon data avail warn = 5 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: mon mgr mkfs grace = 240 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:33:26.910 INFO:teuthology.orchestra.run.trial194.stdout: mon osd reporter subtree level = osd 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: mon osd prime pg temp = true 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:33:26.911 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: # 1m isn't quite enough 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout:[client] 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: rgw cache enabled = true 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: rgw enable ops log = true 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: rgw enable usage log = true 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:33:26.912 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout: debug rbd = 20 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout: debug rbd_mirror = 15 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout: log to stderr = False 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout: rbd default features = 125 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout:[mon.a] 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.0] 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.1] 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.913 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.2] 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.3] 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.4] 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.5] 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.914 INFO:teuthology.orchestra.run.trial194.stdout:[client.mirror.6] 2026-02-22T22:33:26.915 INFO:teuthology.orchestra.run.trial194.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.915 INFO:teuthology.orchestra.run.trial194.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.935 INFO:teuthology.orchestra.run.trial198.stdout:[global] 2026-02-22T22:33:26.935 INFO:teuthology.orchestra.run.trial198.stdout: chdir = "" 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/$cluster-$name.pid 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: auth supported = cephx 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: filestore xattr use omap = true 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: mon clock drift allowed = 1.000 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: osd crush chooseleaf type = 0 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: auth debug = true 2026-02-22T22:33:26.936 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: ms die on old message = true 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: ms die on bug = true 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: mon max pg per osd = 10000 # >= luminous 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: mon pg warn max object skew = 0 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: # disable pg_autoscaler by default for new pools 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: osd_pool_default_pg_autoscale_mode = off 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: osd pool default size = 2 2026-02-22T22:33:26.937 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon osd allow primary affinity = true 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon osd allow pg remap = true 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on legacy crush tunables = false 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on crush straw calc version zero = false 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on no sortbitwise = false 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on osd down out interval zero = false 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on too few osds = false 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_pool_pg_num_not_power_of_two = false 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_pool_no_redundancy = false 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: mon_allow_pool_size_one = true 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.938 INFO:teuthology.orchestra.run.trial198.stdout: osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: osd default data pool replay window = 5 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: mon allow pool delete = true 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: mon cluster log file level = debug 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: debug asserts on shutdown = true 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: mon health detail to clog = false 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: mon host = 10.20.193.198 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: mon client directed command retry = 5 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: mon warn on pool no app = False 2026-02-22T22:33:26.939 INFO:teuthology.orchestra.run.trial198.stdout: ms inject socket failures = 5000 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: fsid = 3163850b-09a4-4a3d-86d6-c3514600e52a 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout:[osd] 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: osd journal size = 100 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: osd scrub load threshold = 5.0 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: osd scrub max interval = 600 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: osd mclock profile = high_recovery_ops 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: osd mclock skip benchmark = true 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: osd recover clone overlap = true 2026-02-22T22:33:26.940 INFO:teuthology.orchestra.run.trial198.stdout: osd recovery max chunk = 1048576 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: osd debug shutdown = true 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: osd debug op order = true 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: osd debug verify stray on activate = true 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: osd debug trim objects = true 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: osd open classes on start = true 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: osd debug pg log writeout = true 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: osd deep scrub update digest min age = 30 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.941 INFO:teuthology.orchestra.run.trial198.stdout: osd map max advance = 10 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: journal zero on create = true 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: filestore ondisk finisher threads = 3 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: filestore apply finisher threads = 3 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: bdev debug aio = true 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: osd debug misdirected ops = true 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: bdev async discard = True 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: bdev enable discard = True 2026-02-22T22:33:26.942 INFO:teuthology.orchestra.run.trial198.stdout: bluestore block size = 96636764160 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: bluestore fsck on mount = True 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: debug bluefs = 1/20 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: debug bluestore = 1/20 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: debug ms = 1 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: debug osd = 20 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: debug rocksdb = 4/10 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: mon osd backfillfull_ratio = 0.85 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: mon osd full ratio = 0.9 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: mon osd nearfull ratio = 0.8 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: osd failsafe full ratio = 0.95 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: osd memory target = 1610612736 2026-02-22T22:33:26.943 INFO:teuthology.orchestra.run.trial198.stdout: osd objectstore = bluestore 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout:[mgr] 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: debug ms = 1 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: debug mgr = 20 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: debug mon = 20 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: debug auth = 20 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: mgr/telemetry/nag = false 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout:[mon] 2026-02-22T22:33:26.944 INFO:teuthology.orchestra.run.trial198.stdout: debug ms = 1 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: debug mon = 20 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: debug paxos = 20 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: debug auth = 20 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: mon data avail warn = 5 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: mon mgr mkfs grace = 240 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: mon reweight min pgs per osd = 4 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: mon osd reporter subtree level = osd 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: mon osd prime pg temp = true 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: mon reweight min bytes per osd = 10 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: # rotate auth tickets quickly to exercise renewal paths 2026-02-22T22:33:26.945 INFO:teuthology.orchestra.run.trial198.stdout: auth mon ticket ttl = 660 # 11m 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: auth service ticket ttl = 240 # 4m 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: # don't complain about insecure global_id in the test suite 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_insecure_global_id_reclaim = false 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_insecure_global_id_reclaim_allowed = false 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: # 1m isn't quite enough 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: mon_down_mkfs_grace = 2m 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: mon_warn_on_filestore_osds = false 2026-02-22T22:33:26.946 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout:[client] 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: rgw cache enabled = true 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: rgw enable ops log = true 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: rgw enable usage log = true 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: log file = /var/log/ceph/$cluster-$name.$pid.log 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/$cluster-$name.$pid.asok 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: debug rbd = 20 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: debug rbd_mirror = 15 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: log to stderr = False 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout: rbd default features = 125 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout:[mon.a] 2026-02-22T22:33:26.947 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.0] 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.1] 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.2] 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.3] 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.948 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.4] 2026-02-22T22:33:26.949 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.949 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.949 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.5] 2026-02-22T22:33:26.949 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.949 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.949 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.6] 2026-02-22T22:33:26.949 INFO:teuthology.orchestra.run.trial198.stdout: admin socket = /var/run/ceph/rbd-mirror.$cluster-$name.asok 2026-02-22T22:33:26.949 INFO:teuthology.orchestra.run.trial198.stdout: pid file = /var/run/ceph/rbd-mirror.$cluster-$name.pid 2026-02-22T22:33:26.950 INFO:tasks.ceph:Creating admin key on cluster2.mon.a... 2026-02-22T22:33:26.951 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --gen-key --name=client.admin --cap mon 'allow *' --cap osd 'allow *' --cap mds 'allow *' --cap mgr 'allow *' /etc/ceph/cluster2.keyring 2026-02-22T22:33:27.034 INFO:tasks.ceph:Copying monmap to all nodes... 2026-02-22T22:33:27.034 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:27.035 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.keyring of=/dev/stdout 2026-02-22T22:33:27.090 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:27.090 DEBUG:teuthology.orchestra.run.trial198:> dd if=/home/ubuntu/cephtest/cluster2.monmap of=/dev/stdout 2026-02-22T22:33:27.147 INFO:tasks.ceph:Sending monmap to node ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:33:27.147 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:27.147 DEBUG:teuthology.orchestra.run.trial194:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T22:33:27.147 DEBUG:teuthology.orchestra.run.trial194:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:33:27.181 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:27.181 DEBUG:teuthology.orchestra.run.trial194:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:33:27.238 INFO:tasks.ceph:Sending monmap to node ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:33:27.238 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:27.239 DEBUG:teuthology.orchestra.run.trial198:> sudo dd of=/etc/ceph/cluster2.keyring 2026-02-22T22:33:27.239 DEBUG:teuthology.orchestra.run.trial198:> sudo chmod 0644 /etc/ceph/cluster2.keyring 2026-02-22T22:33:27.273 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:27.273 DEBUG:teuthology.orchestra.run.trial198:> dd of=/home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:33:27.329 INFO:tasks.ceph:Setting up mon nodes... 2026-02-22T22:33:27.329 INFO:tasks.ceph:Setting up mgr nodes... 2026-02-22T22:33:27.329 DEBUG:teuthology.orchestra.run.trial194:> sudo mkdir -p /var/lib/ceph/mgr/cluster2-x && sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=mgr.x /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T22:33:27.371 INFO:teuthology.orchestra.run.trial194.stdout:creating /var/lib/ceph/mgr/cluster2-x/keyring 2026-02-22T22:33:27.374 INFO:tasks.ceph:Setting up mds nodes... 2026-02-22T22:33:27.374 INFO:tasks.ceph_client:Setting up client nodes... 2026-02-22T22:33:27.374 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.0 /etc/ceph/cluster2.client.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.0.keyring 2026-02-22T22:33:27.448 INFO:teuthology.orchestra.run.trial194.stdout:creating /etc/ceph/cluster2.client.0.keyring 2026-02-22T22:33:27.460 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror /etc/ceph/cluster2.client.mirror.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T22:33:27.497 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.client.mirror.keyring 2026-02-22T22:33:27.509 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.0 /etc/ceph/cluster2.client.mirror.0.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T22:33:27.585 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.client.mirror.0.keyring 2026-02-22T22:33:27.596 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.1 /etc/ceph/cluster2.client.mirror.1.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T22:33:27.670 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.client.mirror.1.keyring 2026-02-22T22:33:27.682 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.2 /etc/ceph/cluster2.client.mirror.2.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T22:33:27.756 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.client.mirror.2.keyring 2026-02-22T22:33:27.768 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.3 /etc/ceph/cluster2.client.mirror.3.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T22:33:27.842 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.client.mirror.3.keyring 2026-02-22T22:33:27.852 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.4 /etc/ceph/cluster2.client.mirror.4.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T22:33:27.927 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.client.mirror.4.keyring 2026-02-22T22:33:27.937 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.5 /etc/ceph/cluster2.client.mirror.5.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T22:33:28.010 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.client.mirror.5.keyring 2026-02-22T22:33:28.022 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool --create-keyring --gen-key --name=client.mirror.6 /etc/ceph/cluster2.client.mirror.6.keyring && sudo chmod 0644 /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T22:33:28.096 INFO:teuthology.orchestra.run.trial198.stdout:creating /etc/ceph/cluster2.client.mirror.6.keyring 2026-02-22T22:33:28.106 INFO:tasks.ceph:Running mkfs on osd nodes... 2026-02-22T22:33:28.106 INFO:tasks.ceph:ctx.disk_config.remote_to_roles_to_dev: {Remote(name='ubuntu@trial194.front.sepia.ceph.com'): {'cluster1.osd.0': '/dev/vg_nvme/lv_1', 'cluster1.osd.1': '/dev/vg_nvme/lv_2', 'cluster1.osd.2': '/dev/vg_nvme/lv_3'}, Remote(name='ubuntu@trial198.front.sepia.ceph.com'): {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'}} 2026-02-22T22:33:28.107 DEBUG:teuthology.orchestra.run.trial198:> sudo mkdir -p /var/lib/ceph/osd/cluster2-0 2026-02-22T22:33:28.171 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:33:28.171 INFO:tasks.ceph:role: cluster2.osd.0 2026-02-22T22:33:28.171 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_1 on ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:33:28.171 DEBUG:teuthology.orchestra.run.trial198:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_1 2026-02-22T22:33:28.236 INFO:teuthology.orchestra.run.trial198.stdout:meta-data=/dev/vg_nvme/lv_1 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:33:28.236 INFO:teuthology.orchestra.run.trial198.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:33:28.236 INFO:teuthology.orchestra.run.trial198.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:33:28.236 INFO:teuthology.orchestra.run.trial198.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:33:28.236 INFO:teuthology.orchestra.run.trial198.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:33:28.236 INFO:teuthology.orchestra.run.trial198.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:33:28.236 INFO:teuthology.orchestra.run.trial198.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:33:28.237 INFO:teuthology.orchestra.run.trial198.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:33:28.237 INFO:teuthology.orchestra.run.trial198.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:33:28.237 INFO:teuthology.orchestra.run.trial198.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:33:28.804 INFO:teuthology.orchestra.run.trial198.stdout:Discarding blocks...Done. 2026-02-22T22:33:28.806 INFO:tasks.ceph:mount /dev/vg_nvme/lv_1 on ubuntu@trial198.front.sepia.ceph.com -o noatime 2026-02-22T22:33:28.807 DEBUG:teuthology.orchestra.run.trial198:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_1 /var/lib/ceph/osd/cluster2-0 2026-02-22T22:33:28.876 DEBUG:teuthology.orchestra.run.trial198:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-0 2026-02-22T22:33:28.943 DEBUG:teuthology.orchestra.run.trial198:> sudo mkdir -p /var/lib/ceph/osd/cluster2-1 2026-02-22T22:33:29.009 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:33:29.009 INFO:tasks.ceph:role: cluster2.osd.1 2026-02-22T22:33:29.009 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_2 on ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:33:29.010 DEBUG:teuthology.orchestra.run.trial198:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_2 2026-02-22T22:33:29.074 INFO:teuthology.orchestra.run.trial198.stdout:meta-data=/dev/vg_nvme/lv_2 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:33:29.074 INFO:teuthology.orchestra.run.trial198.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:33:29.074 INFO:teuthology.orchestra.run.trial198.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:33:29.074 INFO:teuthology.orchestra.run.trial198.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:33:29.074 INFO:teuthology.orchestra.run.trial198.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:33:29.074 INFO:teuthology.orchestra.run.trial198.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:33:29.074 INFO:teuthology.orchestra.run.trial198.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:33:29.074 INFO:teuthology.orchestra.run.trial198.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:33:29.075 INFO:teuthology.orchestra.run.trial198.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:33:29.075 INFO:teuthology.orchestra.run.trial198.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:33:29.633 INFO:teuthology.orchestra.run.trial198.stdout:Discarding blocks...Done. 2026-02-22T22:33:29.635 INFO:tasks.ceph:mount /dev/vg_nvme/lv_2 on ubuntu@trial198.front.sepia.ceph.com -o noatime 2026-02-22T22:33:29.636 DEBUG:teuthology.orchestra.run.trial198:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_2 /var/lib/ceph/osd/cluster2-1 2026-02-22T22:33:29.704 DEBUG:teuthology.orchestra.run.trial198:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-1 2026-02-22T22:33:29.771 DEBUG:teuthology.orchestra.run.trial198:> sudo mkdir -p /var/lib/ceph/osd/cluster2-2 2026-02-22T22:33:29.836 INFO:tasks.ceph:roles_to_devs: {'cluster2.osd.0': '/dev/vg_nvme/lv_1', 'cluster2.osd.1': '/dev/vg_nvme/lv_2', 'cluster2.osd.2': '/dev/vg_nvme/lv_3'} 2026-02-22T22:33:29.836 INFO:tasks.ceph:role: cluster2.osd.2 2026-02-22T22:33:29.836 INFO:tasks.ceph:['mkfs.xfs', '-f', '-i', 'size=2048'] on /dev/vg_nvme/lv_3 on ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:33:29.836 DEBUG:teuthology.orchestra.run.trial198:> yes | sudo mkfs.xfs -f -i size=2048 /dev/vg_nvme/lv_3 2026-02-22T22:33:29.902 INFO:teuthology.orchestra.run.trial198.stdout:meta-data=/dev/vg_nvme/lv_3 isize=2048 agcount=4, agsize=45875200 blks 2026-02-22T22:33:29.902 INFO:teuthology.orchestra.run.trial198.stdout: = sectsz=4096 attr=2, projid32bit=1 2026-02-22T22:33:29.902 INFO:teuthology.orchestra.run.trial198.stdout: = crc=1 finobt=1, sparse=1, rmapbt=0 2026-02-22T22:33:29.902 INFO:teuthology.orchestra.run.trial198.stdout: = reflink=1 bigtime=1 inobtcount=1 nrext64=0 2026-02-22T22:33:29.902 INFO:teuthology.orchestra.run.trial198.stdout:data = bsize=4096 blocks=183500800, imaxpct=25 2026-02-22T22:33:29.903 INFO:teuthology.orchestra.run.trial198.stdout: = sunit=0 swidth=0 blks 2026-02-22T22:33:29.903 INFO:teuthology.orchestra.run.trial198.stdout:naming =version 2 bsize=4096 ascii-ci=0, ftype=1 2026-02-22T22:33:29.903 INFO:teuthology.orchestra.run.trial198.stdout:log =internal log bsize=4096 blocks=89600, version=2 2026-02-22T22:33:29.903 INFO:teuthology.orchestra.run.trial198.stdout: = sectsz=4096 sunit=1 blks, lazy-count=1 2026-02-22T22:33:29.903 INFO:teuthology.orchestra.run.trial198.stdout:realtime =none extsz=4096 blocks=0, rtextents=0 2026-02-22T22:33:30.459 INFO:teuthology.orchestra.run.trial198.stdout:Discarding blocks...Done. 2026-02-22T22:33:30.461 INFO:tasks.ceph:mount /dev/vg_nvme/lv_3 on ubuntu@trial198.front.sepia.ceph.com -o noatime 2026-02-22T22:33:30.461 DEBUG:teuthology.orchestra.run.trial198:> sudo mount -t xfs -o noatime /dev/vg_nvme/lv_3 /var/lib/ceph/osd/cluster2-2 2026-02-22T22:33:30.528 DEBUG:teuthology.orchestra.run.trial198:> sudo /sbin/restorecon /var/lib/ceph/osd/cluster2-2 2026-02-22T22:33:30.596 DEBUG:teuthology.orchestra.run.trial198:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 0 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:33:30.675 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:30.674+0000 7fc694218780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-0/keyring: can't open /var/lib/ceph/osd/cluster2-0/keyring: (2) No such file or directory 2026-02-22T22:33:30.675 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:30.675+0000 7fc694218780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-0/keyring 2026-02-22T22:33:30.675 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:30.675+0000 7fc694218780 -1 bdev(0x55cfc5317800 /var/lib/ceph/osd/cluster2-0/block) open stat got: (1) Operation not permitted 2026-02-22T22:33:30.675 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:30.675+0000 7fc694218780 -1 bluestore(/var/lib/ceph/osd/cluster2-0) _read_fsid unparsable uuid 2026-02-22T22:33:34.569 DEBUG:teuthology.orchestra.run.trial198:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-0 2026-02-22T22:33:34.635 DEBUG:teuthology.orchestra.run.trial198:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 1 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:33:34.711 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:34.711+0000 7f3db14d9780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-1/keyring: can't open /var/lib/ceph/osd/cluster2-1/keyring: (2) No such file or directory 2026-02-22T22:33:34.712 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:34.712+0000 7f3db14d9780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-1/keyring 2026-02-22T22:33:34.712 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:34.712+0000 7f3db14d9780 -1 bdev(0x55855c925800 /var/lib/ceph/osd/cluster2-1/block) open stat got: (1) Operation not permitted 2026-02-22T22:33:34.712 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:34.712+0000 7f3db14d9780 -1 bluestore(/var/lib/ceph/osd/cluster2-1) _read_fsid unparsable uuid 2026-02-22T22:33:38.603 DEBUG:teuthology.orchestra.run.trial198:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-1 2026-02-22T22:33:38.669 DEBUG:teuthology.orchestra.run.trial198:> sudo MALLOC_CHECK_=3 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-osd --no-mon-config --cluster cluster2 --mkfs --mkkey -i 2 --monmap /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:33:38.747 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:38.747+0000 7fbf1a20f780 -1 auth: error reading file: /var/lib/ceph/osd/cluster2-2/keyring: can't open /var/lib/ceph/osd/cluster2-2/keyring: (2) No such file or directory 2026-02-22T22:33:38.747 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:38.747+0000 7fbf1a20f780 -1 created new key in keyring /var/lib/ceph/osd/cluster2-2/keyring 2026-02-22T22:33:38.747 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:38.747+0000 7fbf1a20f780 -1 bdev(0x5593715ab800 /var/lib/ceph/osd/cluster2-2/block) open stat got: (1) Operation not permitted 2026-02-22T22:33:38.747 INFO:teuthology.orchestra.run.trial198.stderr:2026-02-22T22:33:38.747+0000 7fbf1a20f780 -1 bluestore(/var/lib/ceph/osd/cluster2-2) _read_fsid unparsable uuid 2026-02-22T22:33:42.638 DEBUG:teuthology.orchestra.run.trial198:> sudo chown -R ceph:ceph /var/lib/ceph/osd/cluster2-2 2026-02-22T22:33:42.704 INFO:tasks.ceph:Reading keys from all nodes... 2026-02-22T22:33:42.704 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:42.705 DEBUG:teuthology.orchestra.run.trial194:> sudo dd if=/var/lib/ceph/mgr/cluster2-x/keyring of=/dev/stdout 2026-02-22T22:33:42.732 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:42.733 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/var/lib/ceph/osd/cluster2-0/keyring of=/dev/stdout 2026-02-22T22:33:42.768 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:42.768 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/var/lib/ceph/osd/cluster2-1/keyring of=/dev/stdout 2026-02-22T22:33:42.832 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:42.832 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/var/lib/ceph/osd/cluster2-2/keyring of=/dev/stdout 2026-02-22T22:33:42.896 DEBUG:teuthology.orchestra.run.trial194:> set -ex 2026-02-22T22:33:42.896 DEBUG:teuthology.orchestra.run.trial194:> dd if=/etc/ceph/cluster2.client.0.keyring of=/dev/stdout 2026-02-22T22:33:42.915 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:42.915 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.client.mirror.keyring of=/dev/stdout 2026-02-22T22:33:42.952 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:42.952 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.client.mirror.0.keyring of=/dev/stdout 2026-02-22T22:33:43.011 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:43.011 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.client.mirror.1.keyring of=/dev/stdout 2026-02-22T22:33:43.068 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:43.068 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.client.mirror.2.keyring of=/dev/stdout 2026-02-22T22:33:43.126 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:43.126 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.client.mirror.3.keyring of=/dev/stdout 2026-02-22T22:33:43.182 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:43.182 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.client.mirror.4.keyring of=/dev/stdout 2026-02-22T22:33:43.240 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:43.240 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.client.mirror.5.keyring of=/dev/stdout 2026-02-22T22:33:43.296 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:43.296 DEBUG:teuthology.orchestra.run.trial198:> dd if=/etc/ceph/cluster2.client.mirror.6.keyring of=/dev/stdout 2026-02-22T22:33:43.352 INFO:tasks.ceph:Adding keys to all mons... 2026-02-22T22:33:43.353 DEBUG:teuthology.orchestra.run.trial198:> sudo tee -a /etc/ceph/cluster2.keyring 2026-02-22T22:33:43.414 INFO:teuthology.orchestra.run.trial198.stdout:[mgr.x] 2026-02-22T22:33:43.414 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtp6SweFhAA/jKTvHveH9FJK7U0SpZawA== 2026-02-22T22:33:43.414 INFO:teuthology.orchestra.run.trial198.stdout:[osd.0] 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA6hJtpN+FIKBAAWBMsNeDNTwhAlRQ7i/iZJg== 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout:[osd.1] 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA+hJtpaS2AKhAAKoiuS4LhFhZ6uU1kZA8uHA== 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout:[osd.2] 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout: key = AQBChJtpg4qaLBAArwhH2acP8G7lnWTVWRdsMQ== 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout:[client.0] 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtp1gSuGhAAHyV2AanwKrM+yJ8KbQd1Hw== 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror] 2026-02-22T22:33:43.415 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpajK6HRAAbh1z2fxo+V7XJHOkO+gNhg== 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.0] 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpuGXuIhAAwdyiMAEC7WGf9X+oC/vWng== 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.1] 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtp0wMGKBAAV5lCP3cu25PUphgGTn4iIQ== 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.2] 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpay0rLRAAZ/3X4grggk3OPw0sV0gwXA== 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.3] 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpNkRFMhAAXez8lc8YVz7L3m9zJw+hbg== 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.4] 2026-02-22T22:33:43.416 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpnzVPNxAAS00E/4LaNlNnNOxbqZe5pg== 2026-02-22T22:33:43.417 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.5] 2026-02-22T22:33:43.417 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA4hJtpbNy0ABAAQxvz8GryIH8nF5Z56TQxZg== 2026-02-22T22:33:43.417 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.6] 2026-02-22T22:33:43.417 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA4hJtphFfIBRAAipn9e3L1WqPAiO6X+YjSlw== 2026-02-22T22:33:43.418 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=mgr.x --cap mon 'allow profile mgr' --cap osd 'allow *' --cap mds 'allow *' 2026-02-22T22:33:43.500 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.0 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:33:43.580 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.1 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:33:43.664 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=osd.2 --cap mon 'allow profile osd' --cap mgr 'allow profile osd' --cap osd 'allow *' 2026-02-22T22:33:43.742 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:43.823 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:43.904 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.0 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:43.983 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.1 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:44.064 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.2 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:44.147 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.3 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:44.226 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.4 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:44.307 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.5 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:44.387 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-authtool /etc/ceph/cluster2.keyring --name=client.mirror.6 --cap mon 'allow rw' --cap mgr 'allow r' --cap osd 'allow rwx' --cap mds allow 2026-02-22T22:33:44.468 INFO:tasks.ceph:Running mkfs on mon nodes... 2026-02-22T22:33:44.468 DEBUG:teuthology.orchestra.run.trial198:> sudo mkdir -p /var/lib/ceph/mon/cluster2-a 2026-02-22T22:33:44.532 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph-mon --cluster cluster2 --mkfs -i a --monmap /home/ubuntu/cephtest/cluster2.monmap --keyring /etc/ceph/cluster2.keyring 2026-02-22T22:33:44.614 DEBUG:teuthology.orchestra.run.trial198:> sudo chown -R ceph:ceph /var/lib/ceph/mon/cluster2-a 2026-02-22T22:33:44.679 DEBUG:teuthology.orchestra.run.trial198:> rm -- /home/ubuntu/cephtest/cluster2.monmap 2026-02-22T22:33:44.735 INFO:tasks.ceph:Starting mon daemons in cluster cluster2... 2026-02-22T22:33:44.735 INFO:tasks.ceph.cluster2.mon.a:Restarting daemon 2026-02-22T22:33:44.735 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mon -f --cluster cluster2 -i a 2026-02-22T22:33:44.777 INFO:tasks.ceph.cluster2.mon.a:Started 2026-02-22T22:33:44.777 INFO:tasks.ceph:Starting mgr daemons in cluster cluster2... 2026-02-22T22:33:44.778 INFO:tasks.ceph.cluster2.mgr.x:Restarting daemon 2026-02-22T22:33:44.778 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-mgr -f --cluster cluster2 -i x 2026-02-22T22:33:44.781 INFO:tasks.ceph.cluster2.mgr.x:Started 2026-02-22T22:33:44.781 INFO:tasks.ceph:Setting crush tunables to default 2026-02-22T22:33:44.781 DEBUG:teuthology.orchestra.run.trial198:> sudo ceph --cluster cluster2 osd crush tunables default 2026-02-22T22:33:44.882 INFO:teuthology.orchestra.run.trial198.stderr:adjusted tunables profile to default 2026-02-22T22:33:44.890 INFO:tasks.ceph:check_enable_crimson: False 2026-02-22T22:33:44.890 INFO:tasks.ceph:Starting osd daemons in cluster cluster2... 2026-02-22T22:33:44.890 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:44.891 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/var/lib/ceph/osd/cluster2-0/fsid of=/dev/stdout 2026-02-22T22:33:44.916 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:44.916 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/var/lib/ceph/osd/cluster2-1/fsid of=/dev/stdout 2026-02-22T22:33:44.981 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:33:44.981 DEBUG:teuthology.orchestra.run.trial198:> sudo dd if=/var/lib/ceph/osd/cluster2-2/fsid of=/dev/stdout 2026-02-22T22:33:45.046 DEBUG:teuthology.orchestra.run.trial198:> sudo ceph --cluster cluster2 osd new bb727418-5617-4c65-9bbe-c41f7c1c38f8 0 2026-02-22T22:33:45.108 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.105+0000 7f28719ce140 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2026-02-22T22:33:45.192 INFO:teuthology.orchestra.run.trial198.stdout:0 2026-02-22T22:33:45.200 DEBUG:teuthology.orchestra.run.trial198:> sudo ceph --cluster cluster2 osd new 8a10c63e-6aa0-48eb-b9b7-76b88bed0f49 1 2026-02-22T22:33:45.224 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.222+0000 7f28719ce140 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2026-02-22T22:33:45.246 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.244+0000 7f28719ce140 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2026-02-22T22:33:45.272 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.269+0000 7f28719ce140 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2026-02-22T22:33:45.293 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.290+0000 7f28719ce140 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2026-02-22T22:33:45.302 INFO:teuthology.orchestra.run.trial198.stdout:1 2026-02-22T22:33:45.310 DEBUG:teuthology.orchestra.run.trial198:> sudo ceph --cluster cluster2 osd new 95e8a862-6b24-4b97-bf8b-318f2d869686 2 2026-02-22T22:33:45.317 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.315+0000 7f28719ce140 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2026-02-22T22:33:45.368 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.366+0000 7f28719ce140 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2026-02-22T22:33:45.413 INFO:teuthology.orchestra.run.trial198.stdout:2 2026-02-22T22:33:45.421 INFO:tasks.ceph.cluster2.osd.0:Restarting daemon 2026-02-22T22:33:45.421 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 0 2026-02-22T22:33:45.423 INFO:tasks.ceph.cluster2.osd.0:Started 2026-02-22T22:33:45.424 INFO:tasks.ceph.cluster2.osd.1:Restarting daemon 2026-02-22T22:33:45.424 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 1 2026-02-22T22:33:45.426 INFO:tasks.ceph.cluster2.osd.1:Started 2026-02-22T22:33:45.426 INFO:tasks.ceph.cluster2.osd.2:Restarting daemon 2026-02-22T22:33:45.427 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage daemon-helper kill ceph-osd -f --cluster cluster2 -i 2 2026-02-22T22:33:45.428 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:/usr/lib64/python3.9/site-packages/scipy/__init__.py:73: UserWarning: NumPy was imported from a Python sub-interpreter but NumPy does not properly support sub-interpreters. This will likely work for most users but might cause hard to track down issues or subtle bugs. A common user of the rare sub-interpreter feature is wsgi which also allows single-interpreter mode. 2026-02-22T22:33:45.428 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:Improvements in the case of bugs are welcome, but is not on the NumPy roadmap, and full support may require significant effort to achieve. 2026-02-22T22:33:45.428 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr: from numpy import show_config as show_numpy_config 2026-02-22T22:33:45.429 INFO:tasks.ceph.cluster2.osd.2:Started 2026-02-22T22:33:45.429 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:33:45.430 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.427+0000 7f28719ce140 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2026-02-22T22:33:45.454 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.451+0000 7f28719ce140 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2026-02-22T22:33:45.507 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.505+0000 7f28719ce140 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2026-02-22T22:33:45.530 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:33:45.530 INFO:teuthology.orchestra.run.trial198.stdout:{"epoch":5,"fsid":"3163850b-09a4-4a3d-86d6-c3514600e52a","created":"2026-02-22T22:33:44.838704+0000","modified":"2026-02-22T22:33:45.414296+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:33:45.414296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"bb727418-5617-4c65-9bbe-c41f7c1c38f8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8a10c63e-6aa0-48eb-b9b7-76b88bed0f49","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"95e8a862-6b24-4b97-bf8b-318f2d869686","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:45.537 INFO:tasks.ceph.ceph_manager.cluster2:[] 2026-02-22T22:33:45.538 INFO:tasks.ceph:Waiting for OSDs to come up 2026-02-22T22:33:45.548 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.545+0000 7f28719ce140 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2026-02-22T22:33:45.571 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.569+0000 7f28719ce140 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2026-02-22T22:33:45.595 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.592+0000 7f28719ce140 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2026-02-22T22:33:45.619 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.616+0000 7f28719ce140 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2026-02-22T22:33:45.639 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.636+0000 7f28719ce140 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2026-02-22T22:33:45.793 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.790+0000 7f28719ce140 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2026-02-22T22:33:45.840 DEBUG:teuthology.orchestra.run.trial198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:33:45.848 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.845+0000 7f28719ce140 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2026-02-22T22:33:45.868 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.866+0000 7f28719ce140 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2026-02-22T22:33:45.890 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.887+0000 7f28719ce140 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2026-02-22T22:33:45.935 INFO:teuthology.misc.health.trial198.stdout: 2026-02-22T22:33:45.935 INFO:teuthology.misc.health.trial198.stdout:{"epoch":5,"fsid":"3163850b-09a4-4a3d-86d6-c3514600e52a","created":"2026-02-22T22:33:44.838704+0000","modified":"2026-02-22T22:33:45.414296+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:33:45.414296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":2,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"bb727418-5617-4c65-9bbe-c41f7c1c38f8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8a10c63e-6aa0-48eb-b9b7-76b88bed0f49","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"95e8a862-6b24-4b97-bf8b-318f2d869686","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:45.941 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:33:45.942 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.939+0000 7f28719ce140 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:33:45.962 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:45.960+0000 7f28719ce140 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2026-02-22T22:33:46.014 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:46.012+0000 7f28719ce140 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2026-02-22T22:33:46.334 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:46.331+0000 7f28719ce140 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2026-02-22T22:33:46.354 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:46.352+0000 7f28719ce140 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2026-02-22T22:33:46.681 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:46.678+0000 7f28719ce140 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2026-02-22T22:33:47.326 INFO:tasks.ceph.cluster2.osd.0.trial198.stderr:2026-02-22T22:33:47.325+0000 7f69177d0780 -1 Falling back to public interface 2026-02-22T22:33:47.339 INFO:tasks.ceph.cluster2.osd.1.trial198.stderr:2026-02-22T22:33:47.339+0000 7f550ca10780 -1 Falling back to public interface 2026-02-22T22:33:47.347 INFO:tasks.ceph.cluster2.osd.2.trial198.stderr:2026-02-22T22:33:47.347+0000 7fa52f62c780 -1 Falling back to public interface 2026-02-22T22:33:47.688 INFO:tasks.ceph.cluster2.mgr.x.trial194.stderr:2026-02-22T22:33:47.686+0000 7f278e523640 -1 mgr.server handle_report got status from non-daemon mon.a 2026-02-22T22:33:51.791 INFO:tasks.ceph.cluster2.osd.0.trial198.stderr:2026-02-22T22:33:51.791+0000 7f69177d0780 -1 osd.0 0 log_to_monitors true 2026-02-22T22:33:51.842 INFO:tasks.ceph.cluster2.osd.1.trial198.stderr:2026-02-22T22:33:51.842+0000 7f550ca10780 -1 osd.1 0 log_to_monitors true 2026-02-22T22:33:51.880 INFO:tasks.ceph.cluster2.osd.2.trial198.stderr:2026-02-22T22:33:51.880+0000 7fa52f62c780 -1 osd.2 0 log_to_monitors true 2026-02-22T22:33:52.245 DEBUG:teuthology.orchestra.run.trial198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:33:52.397 INFO:teuthology.misc.health.trial198.stdout: 2026-02-22T22:33:52.397 INFO:teuthology.misc.health.trial198.stdout:{"epoch":6,"fsid":"3163850b-09a4-4a3d-86d6-c3514600e52a","created":"2026-02-22T22:33:44.838704+0000","modified":"2026-02-22T22:33:51.844914+0000","last_up_change":"0.000000","last_in_change":"2026-02-22T22:33:45.414296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":3,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":0,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[],"osds":[{"osd":0,"uuid":"bb727418-5617-4c65-9bbe-c41f7c1c38f8","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":1,"uuid":"8a10c63e-6aa0-48eb-b9b7-76b88bed0f49","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]},{"osd":2,"uuid":"95e8a862-6b24-4b97-bf8b-318f2d869686","up":0,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":0,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[]},"cluster_addrs":{"addrvec":[]},"heartbeat_back_addrs":{"addrvec":[]},"heartbeat_front_addrs":{"addrvec":[]},"public_addr":"(unrecognized address family 0)/0","cluster_addr":"(unrecognized address family 0)/0","heartbeat_back_addr":"(unrecognized address family 0)/0","heartbeat_front_addr":"(unrecognized address family 0)/0","state":["exists","new"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":0,"old_weight":0,"last_purged_snaps_scrub":"0.000000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:52.404 DEBUG:teuthology.misc:0 of 3 OSDs are up 2026-02-22T22:33:52.861 INFO:tasks.ceph.cluster2.osd.0.trial198.stderr:2026-02-22T22:33:52.861+0000 7f691373b640 -1 osd.0 0 waiting for initial osdmap 2026-02-22T22:33:52.863 INFO:tasks.ceph.cluster2.osd.0.trial198.stderr:2026-02-22T22:33:52.863+0000 7f690b56e640 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:33:53.854 INFO:tasks.ceph.cluster2.osd.1.trial198.stderr:2026-02-22T22:33:53.854+0000 7f55082d7640 -1 osd.1 0 waiting for initial osdmap 2026-02-22T22:33:53.854 INFO:tasks.ceph.cluster2.osd.2.trial198.stderr:2026-02-22T22:33:53.854+0000 7fa52abba640 -1 osd.2 0 waiting for initial osdmap 2026-02-22T22:33:53.855 INFO:tasks.ceph.cluster2.osd.1.trial198.stderr:2026-02-22T22:33:53.855+0000 7f550010a640 -1 osd.1 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:33:53.856 INFO:tasks.ceph.cluster2.osd.2.trial198.stderr:2026-02-22T22:33:53.855+0000 7fa5229ed640 -1 osd.2 8 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2026-02-22T22:33:58.707 DEBUG:teuthology.orchestra.run.trial198:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:33:58.859 INFO:teuthology.misc.health.trial198.stdout: 2026-02-22T22:33:58.860 INFO:teuthology.misc.health.trial198.stdout:{"epoch":13,"fsid":"3163850b-09a4-4a3d-86d6-c3514600e52a","created":"2026-02-22T22:33:44.838704+0000","modified":"2026-02-22T22:33:58.857595+0000","last_up_change":"2026-02-22T22:33:54.853009+0000","last_in_change":"2026-02-22T22:33:45.414296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:33:56.704026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb727418-5617-4c65-9bbe-c41f7c1c38f8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6800","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6801","nonce":638091441}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6802","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6803","nonce":638091441}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6806","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6807","nonce":638091441}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6804","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6805","nonce":638091441}]},"public_addr":"10.20.193.198:6801/638091441","cluster_addr":"10.20.193.198:6803/638091441","heartbeat_back_addr":"10.20.193.198:6807/638091441","heartbeat_front_addr":"10.20.193.198:6805/638091441","state":["exists","up"]},{"osd":1,"uuid":"8a10c63e-6aa0-48eb-b9b7-76b88bed0f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":11,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6808","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6809","nonce":3890729408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6810","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6811","nonce":3890729408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6814","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6815","nonce":3890729408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6812","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6813","nonce":3890729408}]},"public_addr":"10.20.193.198:6809/3890729408","cluster_addr":"10.20.193.198:6811/3890729408","heartbeat_back_addr":"10.20.193.198:6815/3890729408","heartbeat_front_addr":"10.20.193.198:6813/3890729408","state":["exists","up"]},{"osd":2,"uuid":"95e8a862-6b24-4b97-bf8b-318f2d869686","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6816","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6817","nonce":2341346929}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6818","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6819","nonce":2341346929}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6822","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6823","nonce":2341346929}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6820","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6821","nonce":2341346929}]},"public_addr":"10.20.193.198:6817/2341346929","cluster_addr":"10.20.193.198:6819/2341346929","heartbeat_back_addr":"10.20.193.198:6823/2341346929","heartbeat_front_addr":"10.20.193.198:6821/2341346929","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.789007+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.886670+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.844872+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:33:58.867 DEBUG:teuthology.misc:3 of 3 OSDs are up 2026-02-22T22:33:58.867 INFO:tasks.ceph:Creating RBD pool 2026-02-22T22:33:58.867 DEBUG:teuthology.orchestra.run.trial198:> sudo ceph --cluster cluster2 osd pool create rbd 8 2026-02-22T22:33:59.860 INFO:teuthology.orchestra.run.trial198.stderr:pool 'rbd' created 2026-02-22T22:33:59.872 DEBUG:teuthology.orchestra.run.trial198:> rbd --cluster cluster2 pool init rbd 2026-02-22T22:34:02.874 INFO:tasks.ceph:Starting mds daemons in cluster cluster2... 2026-02-22T22:34:02.875 INFO:tasks.ceph:Waiting until cluster2 daemons up and pgs clean... 2026-02-22T22:34:02.875 INFO:tasks.ceph.ceph_manager.cluster2:waiting for mgr available 2026-02-22T22:34:02.875 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 mgr dump --format=json 2026-02-22T22:34:02.875 INFO:tasks.daemonwatchdog.daemon_watchdog:watchdog starting 2026-02-22T22:34:03.059 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:34:03.069 INFO:teuthology.orchestra.run.trial198.stdout:{"epoch":5,"flags":0,"active_gid":4101,"active_name":"x","active_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6826","nonce":1518295162},{"type":"v1","addr":"10.20.193.194:6827","nonce":1518295162}]},"active_addr":"10.20.193.194:6827/1518295162","active_change":"2026-02-22T22:33:46.683565+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[],"modules":["iostat","nfs","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"update_pg_upmap_activity":{"name":"update_pg_upmap_activity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Updates pg_upmap activity stats to be used in `balancer status detail`","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:10.4.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/loki:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.7.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.5","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.51.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/promtail:3.0.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"quay.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_count_max":{"name":"ssh_keepalive_count_max","type":"int","level":"advanced","flags":0,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"How many times ssh connections can fail liveness checks before the host is marked offline","long_desc":"","tags":[],"see_also":[]},"ssh_keepalive_interval":{"name":"ssh_keepalive_interval","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"How often ssh connections are checked for liveness","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":true,"error_string":"","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_requests":{"name":"max_requests","type":"int","level":"advanced","flags":0,"default_value":"500","min":"","max":"","enum_allowed":[],"desc":"Maximum number of requests to keep in memory. When new request comes in, the oldest request will be removed if the number of requests exceeds the max request number. if un-finished request is removed, error message will be logged in the ceph-mgr log.","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"secondary_zone_period_retry_limit":{"name":"secondary_zone_period_retry_limit","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"RGW module period update retry limit for secondary site","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sqlite3_killpoint":{"name":"sqlite3_killpoint","type":"int","level":"dev","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"force_disabled_modules":{},"last_failure_osd_epoch":0,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"10.20.193.194:0","nonce":2845465789}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"10.20.193.194:0","nonce":1332553467}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"10.20.193.194:0","nonce":464110387}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"10.20.193.194:0","nonce":1376879795}]}]} 2026-02-22T22:34:03.075 INFO:tasks.ceph.ceph_manager.cluster2:mgr available! 2026-02-22T22:34:03.075 INFO:tasks.ceph.ceph_manager.cluster2:waiting for all up 2026-02-22T22:34:03.076 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:34:03.233 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:34:03.234 INFO:teuthology.orchestra.run.trial198.stdout:{"epoch":17,"fsid":"3163850b-09a4-4a3d-86d6-c3514600e52a","created":"2026-02-22T22:33:44.838704+0000","modified":"2026-02-22T22:34:02.865483+0000","last_up_change":"2026-02-22T22:33:54.853009+0000","last_in_change":"2026-02-22T22:33:45.414296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:33:56.704026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:33:59.029125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb727418-5617-4c65-9bbe-c41f7c1c38f8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6800","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6801","nonce":638091441}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6802","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6803","nonce":638091441}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6806","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6807","nonce":638091441}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6804","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6805","nonce":638091441}]},"public_addr":"10.20.193.198:6801/638091441","cluster_addr":"10.20.193.198:6803/638091441","heartbeat_back_addr":"10.20.193.198:6807/638091441","heartbeat_front_addr":"10.20.193.198:6805/638091441","state":["exists","up"]},{"osd":1,"uuid":"8a10c63e-6aa0-48eb-b9b7-76b88bed0f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6808","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6809","nonce":3890729408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6810","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6811","nonce":3890729408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6814","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6815","nonce":3890729408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6812","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6813","nonce":3890729408}]},"public_addr":"10.20.193.198:6809/3890729408","cluster_addr":"10.20.193.198:6811/3890729408","heartbeat_back_addr":"10.20.193.198:6815/3890729408","heartbeat_front_addr":"10.20.193.198:6813/3890729408","state":["exists","up"]},{"osd":2,"uuid":"95e8a862-6b24-4b97-bf8b-318f2d869686","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6816","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6817","nonce":2341346929}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6818","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6819","nonce":2341346929}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6822","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6823","nonce":2341346929}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6820","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6821","nonce":2341346929}]},"public_addr":"10.20.193.198:6817/2341346929","cluster_addr":"10.20.193.198:6819/2341346929","heartbeat_back_addr":"10.20.193.198:6823/2341346929","heartbeat_front_addr":"10.20.193.198:6821/2341346929","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.789007+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.886670+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.844872+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:34:03.243 INFO:tasks.ceph.ceph_manager.cluster2:all up! 2026-02-22T22:34:03.243 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:34:03.399 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:34:03.399 INFO:teuthology.orchestra.run.trial198.stdout:{"epoch":17,"fsid":"3163850b-09a4-4a3d-86d6-c3514600e52a","created":"2026-02-22T22:33:44.838704+0000","modified":"2026-02-22T22:34:02.865483+0000","last_up_change":"2026-02-22T22:33:54.853009+0000","last_in_change":"2026-02-22T22:33:45.414296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:33:56.704026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:33:59.029125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb727418-5617-4c65-9bbe-c41f7c1c38f8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6800","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6801","nonce":638091441}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6802","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6803","nonce":638091441}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6806","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6807","nonce":638091441}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6804","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6805","nonce":638091441}]},"public_addr":"10.20.193.198:6801/638091441","cluster_addr":"10.20.193.198:6803/638091441","heartbeat_back_addr":"10.20.193.198:6807/638091441","heartbeat_front_addr":"10.20.193.198:6805/638091441","state":["exists","up"]},{"osd":1,"uuid":"8a10c63e-6aa0-48eb-b9b7-76b88bed0f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6808","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6809","nonce":3890729408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6810","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6811","nonce":3890729408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6814","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6815","nonce":3890729408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6812","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6813","nonce":3890729408}]},"public_addr":"10.20.193.198:6809/3890729408","cluster_addr":"10.20.193.198:6811/3890729408","heartbeat_back_addr":"10.20.193.198:6815/3890729408","heartbeat_front_addr":"10.20.193.198:6813/3890729408","state":["exists","up"]},{"osd":2,"uuid":"95e8a862-6b24-4b97-bf8b-318f2d869686","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":14,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6816","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6817","nonce":2341346929}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6818","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6819","nonce":2341346929}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6822","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6823","nonce":2341346929}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6820","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6821","nonce":2341346929}]},"public_addr":"10.20.193.198:6817/2341346929","cluster_addr":"10.20.193.198:6819/2341346929","heartbeat_back_addr":"10.20.193.198:6823/2341346929","heartbeat_front_addr":"10.20.193.198:6821/2341346929","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.789007+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.886670+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.844872+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[{"pool":2,"snaps":[{"begin":2,"length":1}]}],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:34:03.408 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 flush_pg_stats 2026-02-22T22:34:03.409 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 flush_pg_stats 2026-02-22T22:34:03.409 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 flush_pg_stats 2026-02-22T22:34:03.497 INFO:teuthology.orchestra.run.trial198.stdout:34359738372 2026-02-22T22:34:03.498 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:34:03.499 INFO:teuthology.orchestra.run.trial198.stdout:38654705668 2026-02-22T22:34:03.499 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:34:03.500 INFO:teuthology.orchestra.run.trial198.stdout:38654705669 2026-02-22T22:34:03.501 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:34:03.700 INFO:teuthology.orchestra.run.trial198.stdout:34359738371 2026-02-22T22:34:03.701 INFO:teuthology.orchestra.run.trial198.stdout:38654705667 2026-02-22T22:34:03.710 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738371 for osd.0 2026-02-22T22:34:03.711 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705667 for osd.1 2026-02-22T22:34:03.741 INFO:teuthology.orchestra.run.trial198.stdout:38654705668 2026-02-22T22:34:03.751 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705668 for osd.2 2026-02-22T22:34:04.712 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.0 2026-02-22T22:34:04.712 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.1 2026-02-22T22:34:04.752 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd last-stat-seq osd.2 2026-02-22T22:34:04.876 INFO:teuthology.orchestra.run.trial198.stdout:38654705668 2026-02-22T22:34:04.877 INFO:teuthology.orchestra.run.trial198.stdout:34359738373 2026-02-22T22:34:04.887 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705668 got 38654705668 for osd.1 2026-02-22T22:34:04.887 INFO:tasks.ceph.ceph_manager.cluster2:need seq 34359738372 got 34359738373 for osd.0 2026-02-22T22:34:04.888 DEBUG:teuthology.parallel:result is None 2026-02-22T22:34:04.888 DEBUG:teuthology.parallel:result is None 2026-02-22T22:34:04.913 INFO:teuthology.orchestra.run.trial198.stdout:38654705669 2026-02-22T22:34:04.923 INFO:tasks.ceph.ceph_manager.cluster2:need seq 38654705669 got 38654705669 for osd.2 2026-02-22T22:34:04.923 DEBUG:teuthology.parallel:result is None 2026-02-22T22:34:04.923 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:34:04.923 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:34:05.119 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:34:05.120 INFO:teuthology.orchestra.run.trial198.stderr:dumped all 2026-02-22T22:34:05.127 INFO:teuthology.orchestra.run.trial198.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2026-02-22T22:34:04.686980+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82480,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033040,"statfs":{"total":289910292480,"available":289825832960,"internally_reserved":0,"allocated":1622016,"data_stored":1302985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.825220"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.868342+0000","last_change":"2026-02-22T22:34:02.868487+0000","last_active":"2026-02-22T22:34:02.868342+0000","last_peered":"2026-02-22T22:34:02.868342+0000","last_clean":"2026-02-22T22:34:02.868342+0000","last_became_active":"2026-02-22T22:34:00.867294+0000","last_became_peered":"2026-02-22T22:34:00.867294+0000","last_unstale":"2026-02-22T22:34:02.868342+0000","last_undegraded":"2026-02-22T22:34:02.868342+0000","last_fullsized":"2026-02-22T22:34:02.868342+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:49.554408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094867900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.868037+0000","last_change":"2026-02-22T22:34:02.868266+0000","last_active":"2026-02-22T22:34:02.868037+0000","last_peered":"2026-02-22T22:34:02.868037+0000","last_clean":"2026-02-22T22:34:02.868037+0000","last_became_active":"2026-02-22T22:34:00.867282+0000","last_became_peered":"2026-02-22T22:34:00.867282+0000","last_unstale":"2026-02-22T22:34:02.868037+0000","last_undegraded":"2026-02-22T22:34:02.868037+0000","last_fullsized":"2026-02-22T22:34:02.868037+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:32:35.739961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072328099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.867989+0000","last_change":"2026-02-22T22:34:02.868262+0000","last_active":"2026-02-22T22:34:02.867989+0000","last_peered":"2026-02-22T22:34:02.867989+0000","last_clean":"2026-02-22T22:34:02.867989+0000","last_became_active":"2026-02-22T22:34:00.867317+0000","last_became_peered":"2026-02-22T22:34:00.867317+0000","last_unstale":"2026-02-22T22:34:02.867989+0000","last_undegraded":"2026-02-22T22:34:02.867989+0000","last_fullsized":"2026-02-22T22:34:02.867989+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:39:15.990906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075292700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.867741+0000","last_change":"2026-02-22T22:34:02.867946+0000","last_active":"2026-02-22T22:34:02.867741+0000","last_peered":"2026-02-22T22:34:02.867741+0000","last_clean":"2026-02-22T22:34:02.867741+0000","last_became_active":"2026-02-22T22:34:00.867325+0000","last_became_peered":"2026-02-22T22:34:00.867325+0000","last_unstale":"2026-02-22T22:34:02.867741+0000","last_undegraded":"2026-02-22T22:34:02.867741+0000","last_fullsized":"2026-02-22T22:34:02.867741+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:29:12.873846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043021699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.869616+0000","last_change":"2026-02-22T22:34:02.869362+0000","last_active":"2026-02-22T22:34:02.869616+0000","last_peered":"2026-02-22T22:34:02.869616+0000","last_clean":"2026-02-22T22:34:02.869616+0000","last_became_active":"2026-02-22T22:34:00.867085+0000","last_became_peered":"2026-02-22T22:34:00.867085+0000","last_unstale":"2026-02-22T22:34:02.869616+0000","last_undegraded":"2026-02-22T22:34:02.869616+0000","last_fullsized":"2026-02-22T22:34:02.869616+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:00:12.510734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089501999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:03.052731+0000","last_change":"2026-02-22T22:34:03.052873+0000","last_active":"2026-02-22T22:34:03.052731+0000","last_peered":"2026-02-22T22:34:03.052731+0000","last_clean":"2026-02-22T22:34:03.052731+0000","last_became_active":"2026-02-22T22:34:00.867119+0000","last_became_peered":"2026-02-22T22:34:00.867119+0000","last_unstale":"2026-02-22T22:34:03.052731+0000","last_undegraded":"2026-02-22T22:34:03.052731+0000","last_fullsized":"2026-02-22T22:34:03.052731+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:47:54.266607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025084499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:03.052708+0000","last_change":"2026-02-22T22:34:03.052868+0000","last_active":"2026-02-22T22:34:03.052708+0000","last_peered":"2026-02-22T22:34:03.052708+0000","last_clean":"2026-02-22T22:34:03.052708+0000","last_became_active":"2026-02-22T22:34:00.867120+0000","last_became_peered":"2026-02-22T22:34:00.867120+0000","last_unstale":"2026-02-22T22:34:03.052708+0000","last_undegraded":"2026-02-22T22:34:03.052708+0000","last_fullsized":"2026-02-22T22:34:03.052708+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:02:27.662737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028151199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.867698+0000","last_change":"2026-02-22T22:34:02.867985+0000","last_active":"2026-02-22T22:34:02.867698+0000","last_peered":"2026-02-22T22:34:02.867698+0000","last_clean":"2026-02-22T22:34:02.867698+0000","last_became_active":"2026-02-22T22:34:00.866689+0000","last_became_peered":"2026-02-22T22:34:00.866689+0000","last_unstale":"2026-02-22T22:34:02.867698+0000","last_undegraded":"2026-02-22T22:34:02.867698+0000","last_fullsized":"2026-02-22T22:34:02.867698+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:56:32.650734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050022800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'72","reported_seq":110,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.867296+0000","last_change":"2026-02-22T22:33:58.085668+0000","last_active":"2026-02-22T22:34:02.867296+0000","last_peered":"2026-02-22T22:34:02.867296+0000","last_clean":"2026-02-22T22:34:02.867296+0000","last_became_active":"2026-02-22T22:33:58.084957+0000","last_became_peered":"2026-02-22T22:33:58.084957+0000","last_unstale":"2026-02-22T22:34:02.867296+0000","last_undegraded":"2026-02-22T22:34:02.867296+0000","last_fullsized":"2026-02-22T22:34:02.867296+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_clean_scrub_stamp":"2026-02-22T22:33:56.854877+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:52:32.982688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705669,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27100,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344740,"statfs":{"total":96636764160,"available":96609013760,"internally_reserved":0,"allocated":139264,"data_stored":39554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705668,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27684,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344156,"statfs":{"total":96636764160,"available":96608415744,"internally_reserved":0,"allocated":737280,"data_stored":629930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27696,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344144,"statfs":{"total":96636764160,"available":96608403456,"internally_reserved":0,"allocated":745472,"data_stored":633501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:34:05.128 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:34:05.284 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:34:05.285 INFO:teuthology.orchestra.run.trial198.stderr:dumped all 2026-02-22T22:34:05.292 INFO:teuthology.orchestra.run.trial198.stdout:{"pg_ready":true,"pg_map":{"version":21,"stamp":"2026-02-22T22:34:04.686980+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":82480,"kb_used_data":1584,"kb_used_omap":4,"kb_used_meta":80827,"kb_avail":283033040,"statfs":{"total":289910292480,"available":289825832960,"internally_reserved":0,"allocated":1622016,"data_stored":1302985,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":82767186},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.825220"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.868342+0000","last_change":"2026-02-22T22:34:02.868487+0000","last_active":"2026-02-22T22:34:02.868342+0000","last_peered":"2026-02-22T22:34:02.868342+0000","last_clean":"2026-02-22T22:34:02.868342+0000","last_became_active":"2026-02-22T22:34:00.867294+0000","last_became_peered":"2026-02-22T22:34:00.867294+0000","last_unstale":"2026-02-22T22:34:02.868342+0000","last_undegraded":"2026-02-22T22:34:02.868342+0000","last_fullsized":"2026-02-22T22:34:02.868342+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:49.554408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094867900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.868037+0000","last_change":"2026-02-22T22:34:02.868266+0000","last_active":"2026-02-22T22:34:02.868037+0000","last_peered":"2026-02-22T22:34:02.868037+0000","last_clean":"2026-02-22T22:34:02.868037+0000","last_became_active":"2026-02-22T22:34:00.867282+0000","last_became_peered":"2026-02-22T22:34:00.867282+0000","last_unstale":"2026-02-22T22:34:02.868037+0000","last_undegraded":"2026-02-22T22:34:02.868037+0000","last_fullsized":"2026-02-22T22:34:02.868037+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:32:35.739961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072328099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.867989+0000","last_change":"2026-02-22T22:34:02.868262+0000","last_active":"2026-02-22T22:34:02.867989+0000","last_peered":"2026-02-22T22:34:02.867989+0000","last_clean":"2026-02-22T22:34:02.867989+0000","last_became_active":"2026-02-22T22:34:00.867317+0000","last_became_peered":"2026-02-22T22:34:00.867317+0000","last_unstale":"2026-02-22T22:34:02.867989+0000","last_undegraded":"2026-02-22T22:34:02.867989+0000","last_fullsized":"2026-02-22T22:34:02.867989+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:39:15.990906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075292700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.867741+0000","last_change":"2026-02-22T22:34:02.867946+0000","last_active":"2026-02-22T22:34:02.867741+0000","last_peered":"2026-02-22T22:34:02.867741+0000","last_clean":"2026-02-22T22:34:02.867741+0000","last_became_active":"2026-02-22T22:34:00.867325+0000","last_became_peered":"2026-02-22T22:34:00.867325+0000","last_unstale":"2026-02-22T22:34:02.867741+0000","last_undegraded":"2026-02-22T22:34:02.867741+0000","last_fullsized":"2026-02-22T22:34:02.867741+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:29:12.873846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043021699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.2","version":"17'2","reported_seq":21,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.869616+0000","last_change":"2026-02-22T22:34:02.869362+0000","last_active":"2026-02-22T22:34:02.869616+0000","last_peered":"2026-02-22T22:34:02.869616+0000","last_clean":"2026-02-22T22:34:02.869616+0000","last_became_active":"2026-02-22T22:34:00.867085+0000","last_became_peered":"2026-02-22T22:34:00.867085+0000","last_unstale":"2026-02-22T22:34:02.869616+0000","last_undegraded":"2026-02-22T22:34:02.869616+0000","last_fullsized":"2026-02-22T22:34:02.869616+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:00:12.510734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089501999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:03.052731+0000","last_change":"2026-02-22T22:34:03.052873+0000","last_active":"2026-02-22T22:34:03.052731+0000","last_peered":"2026-02-22T22:34:03.052731+0000","last_clean":"2026-02-22T22:34:03.052731+0000","last_became_active":"2026-02-22T22:34:00.867119+0000","last_became_peered":"2026-02-22T22:34:00.867119+0000","last_unstale":"2026-02-22T22:34:03.052731+0000","last_undegraded":"2026-02-22T22:34:03.052731+0000","last_fullsized":"2026-02-22T22:34:03.052731+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:47:54.266607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025084499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:03.052708+0000","last_change":"2026-02-22T22:34:03.052868+0000","last_active":"2026-02-22T22:34:03.052708+0000","last_peered":"2026-02-22T22:34:03.052708+0000","last_clean":"2026-02-22T22:34:03.052708+0000","last_became_active":"2026-02-22T22:34:00.867120+0000","last_became_peered":"2026-02-22T22:34:00.867120+0000","last_unstale":"2026-02-22T22:34:03.052708+0000","last_undegraded":"2026-02-22T22:34:03.052708+0000","last_fullsized":"2026-02-22T22:34:03.052708+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:02:27.662737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028151199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"2.3","version":"15'1","reported_seq":20,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.867698+0000","last_change":"2026-02-22T22:34:02.867985+0000","last_active":"2026-02-22T22:34:02.867698+0000","last_peered":"2026-02-22T22:34:02.867698+0000","last_clean":"2026-02-22T22:34:02.867698+0000","last_became_active":"2026-02-22T22:34:00.866689+0000","last_became_peered":"2026-02-22T22:34:00.866689+0000","last_unstale":"2026-02-22T22:34:02.867698+0000","last_undegraded":"2026-02-22T22:34:02.867698+0000","last_fullsized":"2026-02-22T22:34:02.867698+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:56:32.650734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050022800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[{"start":"2","length":"1"}]},{"pgid":"1.0","version":"12'72","reported_seq":110,"reported_epoch":17,"state":"active+clean","last_fresh":"2026-02-22T22:34:02.867296+0000","last_change":"2026-02-22T22:33:58.085668+0000","last_active":"2026-02-22T22:34:02.867296+0000","last_peered":"2026-02-22T22:34:02.867296+0000","last_clean":"2026-02-22T22:34:02.867296+0000","last_became_active":"2026-02-22T22:33:58.084957+0000","last_became_peered":"2026-02-22T22:33:58.084957+0000","last_unstale":"2026-02-22T22:34:02.867296+0000","last_undegraded":"2026-02-22T22:34:02.867296+0000","last_fullsized":"2026-02-22T22:34:02.867296+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_clean_scrub_stamp":"2026-02-22T22:33:56.854877+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:52:32.982688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705669,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27100,"kb_used_data":136,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344740,"statfs":{"total":96636764160,"available":96609013760,"internally_reserved":0,"allocated":139264,"data_stored":39554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705668,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27684,"kb_used_data":720,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344156,"statfs":{"total":96636764160,"available":96608415744,"internally_reserved":0,"allocated":737280,"data_stored":629930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738373,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":27696,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":26942,"kb_avail":94344144,"statfs":{"total":96636764160,"available":96608403456,"internally_reserved":0,"allocated":745472,"data_stored":633501,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27589062},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:34:05.294 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:34:05.294 INFO:tasks.ceph:Waiting until ceph cluster cluster2 is healthy... 2026-02-22T22:34:05.294 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy 2026-02-22T22:34:05.294 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 health --format=json 2026-02-22T22:34:05.465 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:34:05.466 INFO:teuthology.orchestra.run.trial198.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2026-02-22T22:34:05.473 INFO:tasks.ceph.ceph_manager.cluster2:wait_until_healthy done 2026-02-22T22:34:05.474 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T22:34:05.487 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T22:34:05.487 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:34:05.487 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:05.686 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.4] 2026-02-22T22:34:05.686 INFO:teuthology.orchestra.run.trial198.stdout: key = AQAOhJtpp1qHLhAAg5eXoXk8YLDsf/D6gQJViQ== 2026-02-22T22:34:05.687 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:05.687 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:05.687 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.4 2026-02-22T22:34:05.696 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:05.896 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.5] 2026-02-22T22:34:05.897 INFO:teuthology.orchestra.run.trial198.stdout: key = AQAOhJtpQ/h5MxAAojxvft2F7Ld6mpzcgRdyLg== 2026-02-22T22:34:05.897 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:05.897 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:05.897 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.5 2026-02-22T22:34:05.907 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:06.103 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.6] 2026-02-22T22:34:06.103 INFO:teuthology.orchestra.run.trial198.stdout: key = AQAOhJtpbcNiOBAASe4wt/GmX9IVa5DQKzskhA== 2026-02-22T22:34:06.103 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:06.104 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:06.104 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.6 2026-02-22T22:34:06.113 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.4 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:06.311 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.4] 2026-02-22T22:34:06.311 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpnzVPNxAAS00E/4LaNlNnNOxbqZe5pg== 2026-02-22T22:34:06.311 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:06.311 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:06.312 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.4 2026-02-22T22:34:06.321 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.5 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:06.516 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.5] 2026-02-22T22:34:06.516 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA4hJtpbNy0ABAAQxvz8GryIH8nF5Z56TQxZg== 2026-02-22T22:34:06.516 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:06.516 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:06.516 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.5 2026-02-22T22:34:06.525 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.6 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:06.724 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.6] 2026-02-22T22:34:06.724 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA4hJtphFfIBRAAipn9e3L1WqPAiO6X+YjSlw== 2026-02-22T22:34:06.725 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:06.725 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:06.725 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.6 2026-02-22T22:34:06.734 INFO:teuthology.run_tasks:Running task exec... 2026-02-22T22:34:06.749 INFO:teuthology.task.exec:Executing custom commands... 2026-02-22T22:34:06.749 INFO:teuthology.task.exec:Running commands on role cluster1.client.mirror.0 host ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:34:06.749 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:06.944 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror] 2026-02-22T22:34:06.944 INFO:teuthology.orchestra.run.trial198.stdout: key = AQAOhJtpB3MhFRAAE8tajg+W0JWSalBkWz8HVA== 2026-02-22T22:34:06.944 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T22:34:06.944 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:06.944 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror 2026-02-22T22:34:06.954 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:07.151 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.0] 2026-02-22T22:34:07.151 INFO:teuthology.orchestra.run.trial198.stdout: key = AQAOhJtp8twuGhAAsJEe9Q4YTv/fZoHG477dCg== 2026-02-22T22:34:07.151 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:07.151 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:07.151 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.0 2026-02-22T22:34:07.161 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:07.360 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.1] 2026-02-22T22:34:07.360 INFO:teuthology.orchestra.run.trial198.stdout: key = AQAOhJtpl71JHxAAcpIhvWrYur3yWt73k4ckRg== 2026-02-22T22:34:07.361 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:07.361 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:07.361 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.1 2026-02-22T22:34:07.370 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:07.569 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.2] 2026-02-22T22:34:07.569 INFO:teuthology.orchestra.run.trial198.stdout: key = AQAOhJtpC8pUJBAAuanczlqktczDGp+5Xpj7jw== 2026-02-22T22:34:07.570 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:07.570 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:07.570 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.2 2026-02-22T22:34:07.578 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster1 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:07.778 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.3] 2026-02-22T22:34:07.778 INFO:teuthology.orchestra.run.trial198.stdout: key = AQAOhJtpZABwKRAAbGOK5dONCZf94eOwdV502g== 2026-02-22T22:34:07.778 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:07.778 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:07.778 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.3 2026-02-22T22:34:07.788 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror mon '"'"'profile rbd-mirror-peer'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:07.983 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror] 2026-02-22T22:34:07.984 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpajK6HRAAbh1z2fxo+V7XJHOkO+gNhg== 2026-02-22T22:34:07.984 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror-peer" 2026-02-22T22:34:07.984 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:07.985 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror 2026-02-22T22:34:07.994 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.0 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:08.191 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.0] 2026-02-22T22:34:08.191 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpuGXuIhAAwdyiMAEC7WGf9X+oC/vWng== 2026-02-22T22:34:08.191 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:08.191 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:08.191 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.0 2026-02-22T22:34:08.201 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.1 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:08.397 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.1] 2026-02-22T22:34:08.397 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtp0wMGKBAAV5lCP3cu25PUphgGTn4iIQ== 2026-02-22T22:34:08.397 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:08.398 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:08.398 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.1 2026-02-22T22:34:08.406 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.2 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:08.601 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.2] 2026-02-22T22:34:08.601 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpay0rLRAAZ/3X4grggk3OPw0sV0gwXA== 2026-02-22T22:34:08.601 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:08.601 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:08.601 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.2 2026-02-22T22:34:08.611 DEBUG:teuthology.orchestra.run.trial198:> sudo TESTDIR=/home/ubuntu/cephtest bash -c 'sudo ceph --cluster cluster2 auth caps client.mirror.3 mon '"'"'profile rbd-mirror'"'"' osd '"'"'profile rbd'"'"'' 2026-02-22T22:34:08.805 INFO:teuthology.orchestra.run.trial198.stdout:[client.mirror.3] 2026-02-22T22:34:08.806 INFO:teuthology.orchestra.run.trial198.stdout: key = AQA3hJtpNkRFMhAAXez8lc8YVz7L3m9zJw+hbg== 2026-02-22T22:34:08.806 INFO:teuthology.orchestra.run.trial198.stdout: caps mon = "profile rbd-mirror" 2026-02-22T22:34:08.806 INFO:teuthology.orchestra.run.trial198.stdout: caps osd = "profile rbd" 2026-02-22T22:34:08.806 INFO:teuthology.orchestra.run.trial198.stderr:updated caps for client.mirror.3 2026-02-22T22:34:08.815 INFO:teuthology.run_tasks:Running task workunit... 2026-02-22T22:34:08.842 INFO:tasks.workunit:Pulling workunits from ref f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:34:08.843 INFO:tasks.workunit:Making a separate scratch dir for every client... 2026-02-22T22:34:08.843 DEBUG:teuthology.orchestra.run.trial198:> stat -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:34:08.859 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:34:08.860 INFO:teuthology.orchestra.run.trial198.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.cluster1.mirror': No such file or directory 2026-02-22T22:34:08.860 DEBUG:teuthology.orchestra.run.trial198:> mkdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:34:08.916 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:34:08.917 DEBUG:teuthology.orchestra.run.trial198:> cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror && mkdir -- client.mirror 2026-02-22T22:34:08.973 INFO:tasks.workunit:timeout=3h 2026-02-22T22:34:08.973 INFO:tasks.workunit:cleanup=True 2026-02-22T22:34:08.974 DEBUG:teuthology.orchestra.run.trial198:> rm -rf /home/ubuntu/cephtest/clone.cluster1.client.mirror && git clone https://github.com/ceph/ceph-ci.git /home/ubuntu/cephtest/clone.cluster1.client.mirror && cd /home/ubuntu/cephtest/clone.cluster1.client.mirror && git checkout f90f4eaebb28b619cf76e80cbbe997656da69579 2026-02-22T22:34:09.035 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:Cloning into '/home/ubuntu/cephtest/clone.cluster1.client.mirror'... 2026-02-22T22:34:28.709 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:Note: switching to 'f90f4eaebb28b619cf76e80cbbe997656da69579'. 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:You are in 'detached HEAD' state. You can look around, make experimental 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:changes and commit them, and you can discard any commits you make in this 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:state without impacting any branches by switching back to a branch. 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:If you want to create a new branch to retain commits you create, you may 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:do so (now or later) by using -c with the switch command. Example: 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: 2026-02-22T22:34:28.710 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: git switch -c 2026-02-22T22:34:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: 2026-02-22T22:34:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:Or undo this operation with: 2026-02-22T22:34:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: 2026-02-22T22:34:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: git switch - 2026-02-22T22:34:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: 2026-02-22T22:34:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:Turn off this advice by setting config variable advice.detachedHead to false 2026-02-22T22:34:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: 2026-02-22T22:34:28.711 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:HEAD is now at f90f4eaebb2 Merge branch 'wip-74712-squid' of https://github.com/idryomov/ceph into wip-yuri6-testing-2026-02-22-1629-squid 2026-02-22T22:34:28.717 DEBUG:teuthology.orchestra.run.trial198:> cd -- /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror 2026-02-22T22:34:28.773 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2026-02-22T22:34:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T22:34:28.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2026-02-22T22:34:28.871 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2026-02-22T22:34:28.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2026-02-22T22:34:28.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/direct_io' 2026-02-22T22:34:28.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T22:34:28.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2026-02-22T22:34:28.950 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/fs' 2026-02-22T22:34:28.953 DEBUG:teuthology.orchestra.run.trial198:> set -ex 2026-02-22T22:34:28.954 DEBUG:teuthology.orchestra.run.trial198:> dd if=/home/ubuntu/cephtest/workunits.list.cluster1.client.mirror of=/dev/stdout 2026-02-22T22:34:29.012 INFO:tasks.workunit:Running workunits matching rbd/rbd_mirror_bootstrap.sh on cluster1.client.mirror... 2026-02-22T22:34:29.014 INFO:tasks.workunit:Running workunit rbd/rbd_mirror_bootstrap.sh... 2026-02-22T22:34:29.014 DEBUG:teuthology.orchestra.run.trial198:workunit test rbd/rbd_mirror_bootstrap.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && cd -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=f90f4eaebb28b619cf76e80cbbe997656da69579 TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster cluster1" CEPH_ID="mirror" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror CEPH_MNT=/home/ubuntu/cephtest/mnt.cluster1.mirror CEPH_ARGS='' MIRROR_POOL_MODE=pool RBD_MIRROR_INSTANCES=1 RBD_MIRROR_MODE=journal RBD_MIRROR_USE_EXISTING_CLUSTER=1 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_bootstrap.sh 2026-02-22T22:34:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ RBD_MIRROR_MANUAL_PEERS=1 2026-02-22T22:34:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ RBD_MIRROR_INSTANCES=1 2026-02-22T22:34:29.072 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_bootstrap.sh 2026-02-22T22:34:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ . /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_helpers.sh 2026-02-22T22:34:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ type xmlstarlet 2026-02-22T22:34:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ XMLSTARLET=xmlstarlet 2026-02-22T22:34:29.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ RBD_MIRROR_INSTANCES=1 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CLUSTER1=cluster1 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CLUSTER2=cluster2 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ PEER_CLUSTER_SUFFIX= 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ POOL=mirror 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ PARENT_POOL=mirror_parent 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ NS1=ns1 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ NS2=ns2 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ TEMPDIR= 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ID=mirror 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ RBD_IMAGE_FEATURES=layering,exclusive-lock,journaling 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ MIRROR_USER_ID_PREFIX=mirror. 2026-02-22T22:34:29.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ RBD_MIRROR_MODE=journal 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ MIRROR_POOL_MODE=pool 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' journal = snapshot ']' 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ export 'CEPH_ARGS=--id mirror' 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS='--id mirror' 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ LAST_MIRROR_INSTANCE=0 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ dirname /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/rbd_mirror_bootstrap.sh 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ readlink -f /home/ubuntu/cephtest/clone.cluster1.client.mirror/qa/workunits/rbd/../../../src 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ROOT=/home/ubuntu/cephtest/clone.cluster1.client.mirror/src 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_BIN=. 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_SRC=. 2026-02-22T22:34:29.077 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' -e CMakeCache.txt ']' 2026-02-22T22:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 -gt 0 ']' 2026-02-22T22:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ setup 2026-02-22T22:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local c 2026-02-22T22:34:29.079 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ trap 'cleanup $?' INT TERM EXIT 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ setup_tempdir 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ mktemp -d 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ TEMPDIR=/tmp/tmp.O4E04S2FH2 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -z 1 ']' 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ setup_pools cluster1 cluster2 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local remote_cluster=cluster2 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mon_map_file 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mon_addr 2026-02-22T22:34:29.080 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local admin_key_file 2026-02-22T22:34:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local uuid 2026-02-22T22:34:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:29.081 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --cluster cluster1 osd pool create mirror 64 64 2026-02-22T22:34:29.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:pool 'mirror' already exists 2026-02-22T22:34:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:29.841 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --cluster cluster1 osd pool create mirror_parent 64 64 2026-02-22T22:34:30.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:pool 'mirror_parent' already exists 2026-02-22T22:34:30.843 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:30.844 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 pool init mirror 2026-02-22T22:34:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:33.804 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 pool init mirror_parent 2026-02-22T22:34:36.814 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:36.814 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:36.814 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 mirror pool enable --site-name cluster1 mirror pool 2026-02-22T22:34:36.844 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent image 2026-02-22T22:34:36.869 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 namespace create mirror/ns1 2026-02-22T22:34:36.894 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 namespace create mirror/ns2 2026-02-22T22:34:36.918 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 namespace create mirror_parent/ns1 2026-02-22T22:34:36.941 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns1 pool 2026-02-22T22:34:36.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 mirror pool enable mirror/ns2 image 2026-02-22T22:34:36.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 mirror pool enable mirror_parent/ns1 pool 2026-02-22T22:34:37.016 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -z 1 ']' 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ setup_pools cluster2 cluster1 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local remote_cluster=cluster1 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mon_map_file 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mon_addr 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local admin_key_file 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local uuid 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:37.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --cluster cluster2 osd pool create mirror 64 64 2026-02-22T22:34:37.899 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:pool 'mirror' already exists 2026-02-22T22:34:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:37.907 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --cluster cluster2 osd pool create mirror_parent 64 64 2026-02-22T22:34:38.913 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:pool 'mirror_parent' already exists 2026-02-22T22:34:38.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:38.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 pool init mirror 2026-02-22T22:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:41.872 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 pool init mirror_parent 2026-02-22T22:34:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:44.878 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 mirror pool enable --site-name cluster2 mirror pool 2026-02-22T22:34:44.906 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent image 2026-02-22T22:34:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 namespace create mirror/ns1 2026-02-22T22:34:44.954 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 namespace create mirror/ns2 2026-02-22T22:34:44.977 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 namespace create mirror_parent/ns1 2026-02-22T22:34:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns1 pool 2026-02-22T22:34:45.025 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 mirror pool enable mirror/ns2 image 2026-02-22T22:34:45.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 mirror pool enable mirror_parent/ns1 pool 2026-02-22T22:34:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -z 1 ']' 2026-02-22T22:34:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ testlog 'TEST: bootstrap cluster2 from cluster1' 2026-02-22T22:34:45.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ tee -a /tmp/tmp.O4E04S2FH2/rbd-mirror.test.log 2026-02-22T22:34:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ date '+%F %T' 2026-02-22T22:34:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo 2026-02-22 22:34:45 TEST: bootstrap cluster2 from cluster1 2026-02-22T22:34:45.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:2026-02-22 22:34:45 TEST: bootstrap cluster2 from cluster1 2026-02-22T22:34:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ TOKEN=/tmp/tmp.O4E04S2FH2/peer-token 2026-02-22T22:34:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ TOKEN_2=/tmp/tmp.O4E04S2FH2/peer-token-2 2026-02-22T22:34:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:45.076 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 mirror pool peer bootstrap create mirror 2026-02-22T22:34:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:45.100 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 mirror pool peer bootstrap create mirror_parent 2026-02-22T22:34:45.123 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmp /tmp/tmp.O4E04S2FH2/peer-token /tmp/tmp.O4E04S2FH2/peer-token-2 2026-02-22T22:34:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:45.124 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 --pool mirror mirror pool peer bootstrap import /tmp/tmp.O4E04S2FH2/peer-token --direction rx-only 2026-02-22T22:34:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:34:45.158 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 --pool mirror_parent mirror pool peer bootstrap import /tmp/tmp.O4E04S2FH2/peer-token --direction rx-tx 2026-02-22T22:34:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ start_mirrors cluster1 2026-02-22T22:34:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:45.195 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:34:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ start_mirror cluster1:0 2026-02-22T22:34:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1:0 2026-02-22T22:34:45.197 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster1:0 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster1 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster1 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:45.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:34:45.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd-mirror --cluster cluster1 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:34:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ start_mirrors cluster2 2026-02-22T22:34:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:45.216 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:34:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ start_mirror cluster2:0 2026-02-22T22:34:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:34:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:34:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:34:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:34:45.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:34:45.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd-mirror --cluster cluster2 --id mirror.0 --rbd-mirror-delete-retry-interval=5 --rbd-mirror-image-state-check-interval=5 --rbd-mirror-journal-poll-age=1 --rbd-mirror-pool-replayers-refresh-interval=5 --debug-rbd=30 --debug-journaler=30 --debug-rbd_mirror=30 --daemonize=true 2026-02-22T22:34:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ testlog 'TEST: verify rx-only direction' 2026-02-22T22:34:45.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ tee -a /tmp/tmp.O4E04S2FH2/rbd-mirror.test.log 2026-02-22T22:34:45.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ date '+%F %T' 2026-02-22T22:34:45.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo 2026-02-22 22:34:45 TEST: verify rx-only direction 2026-02-22T22:34:45.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:2026-02-22 22:34:45 TEST: verify rx-only direction 2026-02-22T22:34:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 --pool mirror mirror pool info --format json 2026-02-22T22:34:45.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.peers[0].direction == "rx-only"' 2026-02-22T22:34:45.261 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 --pool mirror mirror pool info --format json 2026-02-22T22:34:45.262 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.peers | length > 0' 2026-02-22T22:34:45.286 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 --pool mirror mirror pool info --format json 2026-02-22T22:34:45.287 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.peers[0].direction == "tx-only"' 2026-02-22T22:34:45.310 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image_and_enable_mirror cluster1 mirror image1 2026-02-22T22:34:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:45.312 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mode=journal 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image cluster1 mirror image1 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:45.313 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size=128 2026-02-22T22:34:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:45.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/image1 2026-02-22T22:34:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ is_pool_mirror_mode_image mirror 2026-02-22T22:34:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' pool = image ']' 2026-02-22T22:34:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ case "${pool}" in 2026-02-22T22:34:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 1 2026-02-22T22:34:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image_and_enable_mirror cluster1 mirror/ns1 image1 2026-02-22T22:34:45.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mode=journal 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image cluster1 mirror/ns1 image1 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:45.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size=128 2026-02-22T22:34:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:45.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 create --size 128 --image-feature layering,exclusive-lock,journaling mirror/ns1/image1 2026-02-22T22:34:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ is_pool_mirror_mode_image mirror/ns1 2026-02-22T22:34:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' pool = image ']' 2026-02-22T22:34:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ case "${pool}" in 2026-02-22T22:34:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 1 2026-02-22T22:34:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_started cluster2 mirror image1 2026-02-22T22:34:45.379 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_state cluster2 mirror image1 started 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=started 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:45.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:34:46.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_image_replay_state cluster2 mirror image1 started 2026-02-22T22:34:46.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:46.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:46.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:46.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local test_state=started 2026-02-22T22:34:46.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status_result 2026-02-22T22:34:46.380 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local current_state=stopped 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemons cluster2 rbd mirror status mirror/image1 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_instance=cluster2 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance=cluster2 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local loop_instance 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -i state 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:46.382 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 0 2026-02-22T22:34:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:46.383 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:34:46.387 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:46.387 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/image1 2026-02-22T22:34:46.387 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:34:46.387 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:46.387 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:46.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster2 2026-02-22T22:34:46.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:34:46.390 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:34:46.390 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:46.397 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/image1 2026-02-22T22:34:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:Invalid command: unused arguments: ['mirror/image1'] 2026-02-22T22:34:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:34:46.440 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:admin_socket: invalid command 2026-02-22T22:34:46.441 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:46.441 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 1 2026-02-22T22:34:47.442 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:47.442 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:34:47.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:47.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/image1 2026-02-22T22:34:47.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:34:47.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:47.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:47.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:47.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster2 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:34:47.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:47.457 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:47.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:47.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:47.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/image1 2026-02-22T22:34:47.499 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:Invalid command: unused arguments: ['mirror/image1'] 2026-02-22T22:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:admin_socket: invalid command 2026-02-22T22:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:47.500 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 2 2026-02-22T22:34:49.501 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:49.502 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:34:49.505 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/image1 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:49.506 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:34:49.507 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster2 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:34:49.508 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:49.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:49.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:49.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:49.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/image1 2026-02-22T22:34:49.567 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ return 0 2026-02-22T22:34:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:34:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:34:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep -i Replaying 2026-02-22T22:34:49.568 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: "state": "Replaying" 2026-02-22T22:34:49.569 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ current_state=started 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test started = started 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ write_image cluster1 mirror image1 100 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local count=100 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size= 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:34:49.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ size=4096 2026-02-22T22:34:49.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 bench mirror/image1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:34:49.606 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:34:49.618 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:34:49.618 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:elapsed: 0 ops: 100 ops/sec: 8333.47 bytes/sec: 33 MiB/s 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror image1 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster2 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror image1 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster2 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:34:49.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:34:49.826 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster2 mirror image1 2026-02-22T22:34:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:34:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image1 ']' 2026-02-22T22:34:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror/image1' 2026-02-22T22:34:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/image1 2026-02-22T22:34:49.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster2 2026-02-22T22:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster2 2026-02-22T22:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:34:49.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:49.831 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:34:49.831 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:49.831 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/image1 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:49.832 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:49.833 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:49.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:49.841 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:49.841 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:49.842 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/image1 2026-02-22T22:34:49.893 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster1 mirror image1 2026-02-22T22:34:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:49.894 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror image1 '' 2026-02-22T22:34:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:34:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror-image1.status 2026-02-22T22:34:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror-image1.status 2026-02-22T22:34:49.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror/image1 2026-02-22T22:34:49.896 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:49.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=304b4d47-cdec-4fd0-808f-7f6fa559fcc1, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=35], [object_number=2, tag_tid=2, entry_tid=34], [object_number=1, tag_tid=2, entry_tid=33], [object_number=0, tag_tid=2, entry_tid=32]]], state=connected] 2026-02-22T22:34:49.929 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster1 mirror image1 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror image1 '..*' 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:49.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:34:49.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror-image1.status 2026-02-22T22:34:49.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror-image1.status 2026-02-22T22:34:49.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror/image1 2026-02-22T22:34:49.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:49.965 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:49.965 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:49.965 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:49.965 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:49.965 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=304b4d47-cdec-4fd0-808f-7f6fa559fcc1, commit_position=[positions=[[object_number=3, tag_tid=2, entry_tid=35], [object_number=2, tag_tid=2, entry_tid=34], [object_number=1, tag_tid=2, entry_tid=33], [object_number=0, tag_tid=2, entry_tid=32]]], state=connected] 2026-02-22T22:34:49.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:49.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=3, tag_tid=2, entry_tid=35' 2026-02-22T22:34:49.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=2, entry_tid=35' 2026-02-22T22:34:49.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=35' '!=' '' 2026-02-22T22:34:49.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:34:49.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=2, entry_tid=35' = '' 2026-02-22T22:34:49.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_mirror_pos='object_number=3, tag_tid=2, entry_tid=35' 2026-02-22T22:34:49.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:49.968 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:34:49.968 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_tag=2 2026-02-22T22:34:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=35' 2026-02-22T22:34:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:34:49.969 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:49.970 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_tag=2 2026-02-22T22:34:49.971 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:49.971 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:34:49.971 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:49.972 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_entry=101 2026-02-22T22:34:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=2, entry_tid=35' 2026-02-22T22:34:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:34:49.973 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:49.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_entry=35 2026-02-22T22:34:49.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 2 = 2 -a 101 -le 35 2026-02-22T22:34:49.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:34:49.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:34:49.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:34:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster2 mirror image1 2026-02-22T22:34:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:50.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image1 ']' 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror/image1' 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/image1 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster2 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster2 2026-02-22T22:34:50.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:34:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:34:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:50.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/image1 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:34:50.181 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:50.182 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:50.183 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:50.192 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:50.193 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/image1 2026-02-22T22:34:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster1 mirror image1 2026-02-22T22:34:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:50.240 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror image1 '' 2026-02-22T22:34:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:34:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror-image1.status 2026-02-22T22:34:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror-image1.status 2026-02-22T22:34:50.241 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:50.242 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:50.242 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror/image1 2026-02-22T22:34:50.242 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:50.274 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=304b4d47-cdec-4fd0-808f-7f6fa559fcc1, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:50.275 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster1 mirror image1 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror image1 '..*' 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:50.277 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror-image1.status 2026-02-22T22:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror-image1.status 2026-02-22T22:34:50.278 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror/image1 2026-02-22T22:34:50.279 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:50.310 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:50.311 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=304b4d47-cdec-4fd0-808f-7f6fa559fcc1, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:50.312 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror-image1.status 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror image1 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_status_in_pool_dir cluster2 mirror image1 up+replaying primary_position 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:50.314 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:34:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_status_in_pool_dir cluster2 mirror image1 up+replaying primary_position '' 2026-02-22T22:34:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:51.315 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:34:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:34:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:34:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:34:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:34:51.316 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror image status mirror/image1 2026-02-22T22:34:51.346 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='image1: 2026-02-22T22:34:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: global_id: 145b1de2-4c1d-43c7-9292-8fa1ecce51b9 2026-02-22T22:34:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+replaying 2026-02-22T22:34:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:34:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: service: mirror.0 on trial198 2026-02-22T22:34:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:34:48' 2026-02-22T22:34:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:34:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: state: up+replaying 2026-02-22T22:34:51.347 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:34:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:34:51.348 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:34:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^up+' 2026-02-22T22:34:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:34:51.349 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:34:51.350 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: service: mirror.0 on trial198 2026-02-22T22:34:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update 2026-02-22T22:34:51.351 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:48' 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_mirror_pool_status_verbose cluster2 mirror image1 up+replaying '2026-02-22 22:34:48' 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'prev_last_update=2026-02-22 22:34:48' 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:34:51.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format xml 2026-02-22T22:34:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='OKOKOK142594264mirror.0trial19819.2.3-689-gf90f4eaetrueOKimage1145b1de2-4c1d-43c7-9292-8fa1ecce51b9up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42594264mirror.0trial1982026-02-22 22:34:48' 2026-02-22T22:34:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update state 2026-02-22T22:34:51.383 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/last_update' 2026-02-22T22:34:51.386 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:48' 2026-02-22T22:34:51.386 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/state' 2026-02-22T22:34:51.387 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ state=up+replaying 2026-02-22T22:34:51.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:34:51.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep up+replaying 2026-02-22T22:34:51.388 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:34:51.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:51.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:51.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster1 mirror 2026-02-22T22:34:51.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:51.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:51.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:34:51.389 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror --verbose --format json 2026-02-22T22:34:51.421 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"WARNING","daemon_health":"WARNING","image_health":"OK","states":{"replaying":1}},"daemons":[{"service_id":"4270","instance_id":"","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"OK"}],"images":[{"name":"image1","global_id":"145b1de2-4c1d-43c7-9292-8fa1ecce51b9"}]}' 2026-02-22T22:34:51.421 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.states == {"replaying": 1}' 2026-02-22T22:34:51.422 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:51.423 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror 2026-02-22T22:34:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror 2026-02-22T22:34:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:34:51.424 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror --verbose --format json 2026-02-22T22:34:51.452 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":1}},"daemons":[{"service_id":"4259","instance_id":"4264","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"145b1de2-4c1d-43c7-9292-8fa1ecce51b9","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4259","instance_id":"4264","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:48"}]}' 2026-02-22T22:34:51.453 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.states == {"replaying": 1}' 2026-02-22T22:34:51.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_started cluster2 mirror/ns1 image1 2026-02-22T22:34:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_state cluster2 mirror/ns1 image1 started 2026-02-22T22:34:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:51.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:51.456 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:51.456 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=started 2026-02-22T22:34:51.456 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s 2026-02-22T22:34:51.456 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:51.456 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:34:52.457 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_image_replay_state cluster2 mirror/ns1 image1 started 2026-02-22T22:34:52.457 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:52.457 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:52.457 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local test_state=started 2026-02-22T22:34:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status_result 2026-02-22T22:34:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local current_state=stopped 2026-02-22T22:34:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemons cluster2 rbd mirror status mirror/ns1/image1 2026-02-22T22:34:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -i state 2026-02-22T22:34:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_instance=cluster2 2026-02-22T22:34:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:52.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance=cluster2 2026-02-22T22:34:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local loop_instance 2026-02-22T22:34:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:52.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 0 2026-02-22T22:34:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:34:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:52.464 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror/ns1/image1 2026-02-22T22:34:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:34:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:34:52.465 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:52.466 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:34:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2 2026-02-22T22:34:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:34:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:34:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:34:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:34:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:52.467 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:34:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster2 2026-02-22T22:34:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:34:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:34:52.468 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:52.472 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:52.472 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:52.473 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror/ns1/image1 2026-02-22T22:34:52.517 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ return 0 2026-02-22T22:34:52.518 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:34:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:34:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep -i Replaying 2026-02-22T22:34:52.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: "state": "Replaying" 2026-02-22T22:34:52.520 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ current_state=started 2026-02-22T22:34:52.520 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test started = started 2026-02-22T22:34:52.520 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ write_image cluster1 mirror/ns1 image1 100 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local count=100 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size= 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ size=4096 2026-02-22T22:34:52.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 bench mirror/ns1/image1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:34:52.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:34:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:34:52.573 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:elapsed: 0 ops: 100 ops/sec: 5882.45 bytes/sec: 23 MiB/s 2026-02-22T22:34:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror/ns1 image1 2026-02-22T22:34:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster2 2026-02-22T22:34:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:52.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror/ns1 image1 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster2 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:34:52.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster2 mirror/ns1 image1 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image1 ']' 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror/ns1/image1' 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/ns1/image1 2026-02-22T22:34:52.781 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster2 2026-02-22T22:34:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster2 2026-02-22T22:34:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:34:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:34:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:52.782 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/ns1/image1 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:34:52.783 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:52.784 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:52.785 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:52.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:52.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/ns1/image1 2026-02-22T22:34:52.847 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster1 mirror/ns1 image1 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror/ns1 image1 '' 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror/ns1-image1.status 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror/ns1-image1.status 2026-02-22T22:34:52.848 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:52.849 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror/ns1/image1 2026-02-22T22:34:52.850 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:52.882 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=304b4d47-cdec-4fd0-808f-7f6fa559fcc1, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:34:52.883 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster1 mirror/ns1 image1 2026-02-22T22:34:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror/ns1 image1 '..*' 2026-02-22T22:34:52.886 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:34:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror/ns1-image1.status 2026-02-22T22:34:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror/ns1-image1.status 2026-02-22T22:34:52.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror/ns1/image1 2026-02-22T22:34:52.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:52.922 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:52.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=304b4d47-cdec-4fd0-808f-7f6fa559fcc1, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:34:52.924 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:34:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:34:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:34:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:52.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:34:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:52.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_tag=2 2026-02-22T22:34:52.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:52.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:34:52.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:52.928 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_tag=1 2026-02-22T22:34:52.929 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:52.929 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:34:52.929 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:52.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_entry=101 2026-02-22T22:34:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:34:52.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_entry=3 2026-02-22T22:34:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:34:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:34:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:34:52.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:34:53.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster2 mirror/ns1 image1 2026-02-22T22:34:53.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image1 ']' 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror/ns1/image1' 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster2 rbd mirror flush mirror/ns1/image1 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster2 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster2 2026-02-22T22:34:53.135 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:34:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:34:53.136 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror/ns1/image1 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:34:53.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:53.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:34:53.141 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:34:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:53.142 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:53.150 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:53.151 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror/ns1/image1 2026-02-22T22:34:53.197 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster1 mirror/ns1 image1 2026-02-22T22:34:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:53.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror/ns1 image1 '' 2026-02-22T22:34:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:34:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror/ns1-image1.status 2026-02-22T22:34:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror/ns1-image1.status 2026-02-22T22:34:53.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror/ns1/image1 2026-02-22T22:34:53.201 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:53.231 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=304b4d47-cdec-4fd0-808f-7f6fa559fcc1, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:53.232 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:53.233 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster1 mirror/ns1 image1 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror/ns1 image1 '..*' 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:34:53.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror/ns1-image1.status 2026-02-22T22:34:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror/ns1-image1.status 2026-02-22T22:34:53.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror/ns1/image1 2026-02-22T22:34:53.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:53.268 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:53.268 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:53.268 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:53.268 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:53.268 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=304b4d47-cdec-4fd0-808f-7f6fa559fcc1, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:53.269 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_ns1-image1.status 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror/ns1 image1 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_status_in_pool_dir cluster2 mirror/ns1 image1 up+replaying primary_position 2026-02-22T22:34:53.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:53.273 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:53.273 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:53.273 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:34:53.273 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:34:53.273 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:34:53.273 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:53.273 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:34:54.271 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_status_in_pool_dir cluster2 mirror/ns1 image1 up+replaying primary_position '' 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:34:54.272 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror image status mirror/ns1/image1 2026-02-22T22:34:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='image1: 2026-02-22T22:34:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: global_id: 0f350ea5-ba1d-4e0c-8bb1-9bc8f3bc19ad 2026-02-22T22:34:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+replaying 2026-02-22T22:34:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:34:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: service: mirror.0 on trial198 2026-02-22T22:34:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:34:49' 2026-02-22T22:34:54.303 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:34:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: state: up+replaying 2026-02-22T22:34:54.304 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:34:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:34:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:34:54.305 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^up+' 2026-02-22T22:34:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:34:54.306 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:34:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: service: mirror.0 on trial198 2026-02-22T22:34:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update 2026-02-22T22:34:54.307 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:34:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:49' 2026-02-22T22:34:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_mirror_pool_status_verbose cluster2 mirror/ns1 image1 up+replaying '2026-02-22 22:34:49' 2026-02-22T22:34:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror/ns1 2026-02-22T22:34:54.309 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:34:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'prev_last_update=2026-02-22 22:34:49' 2026-02-22T22:34:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:34:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:34:54.310 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror/ns1 --verbose --format xml 2026-02-22T22:34:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='OKOKOK142594264mirror.0trial19819.2.3-689-gf90f4eaetrueOKimage10f350ea5-ba1d-4e0c-8bb1-9bc8f3bc19adup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42594264mirror.0trial1982026-02-22 22:34:49' 2026-02-22T22:34:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update state 2026-02-22T22:34:54.337 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/last_update' 2026-02-22T22:34:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:49' 2026-02-22T22:34:54.339 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/state' 2026-02-22T22:34:54.340 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ state=up+replaying 2026-02-22T22:34:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:34:54.341 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep up+replaying 2026-02-22T22:34:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:34:54.342 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster1 mirror/ns1 2026-02-22T22:34:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:34:54.343 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror/ns1 --verbose --format json 2026-02-22T22:34:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"WARNING","daemon_health":"WARNING","image_health":"OK","states":{"replaying":1}},"daemons":[{"service_id":"4270","instance_id":"","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"OK"}],"images":[{"name":"image1","global_id":"0f350ea5-ba1d-4e0c-8bb1-9bc8f3bc19ad"}]}' 2026-02-22T22:34:54.374 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.states == {"replaying": 1}' 2026-02-22T22:34:54.376 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror/ns1 2026-02-22T22:34:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror/ns1 2026-02-22T22:34:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:34:54.377 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror/ns1 --verbose --format json 2026-02-22T22:34:54.405 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":1}},"daemons":[{"service_id":"4259","instance_id":"4264","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"0f350ea5-ba1d-4e0c-8bb1-9bc8f3bc19ad","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4259","instance_id":"4264","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:49"}]}' 2026-02-22T22:34:54.406 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.states == {"replaying": 1}' 2026-02-22T22:34:54.407 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ testlog 'TEST: verify rx-tx direction' 2026-02-22T22:34:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ tee -a /tmp/tmp.O4E04S2FH2/rbd-mirror.test.log 2026-02-22T22:34:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ date '+%F %T' 2026-02-22T22:34:54.408 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo 2026-02-22 22:34:54 TEST: verify rx-tx direction 2026-02-22T22:34:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:2026-02-22 22:34:54 TEST: verify rx-tx direction 2026-02-22T22:34:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 --pool mirror_parent mirror pool info --format json 2026-02-22T22:34:54.409 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.peers[0].direction == "rx-tx"' 2026-02-22T22:34:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 --pool mirror_parent mirror pool info --format json 2026-02-22T22:34:54.434 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.peers[0].direction == "rx-tx"' 2026-02-22T22:34:54.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:34:54.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image_and_enable_mirror cluster1 mirror_parent image1 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mode=journal 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image cluster1 mirror_parent image1 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size=128 2026-02-22T22:34:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/image1 2026-02-22T22:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-02-22T22:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' pool = image ']' 2026-02-22T22:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ case "${pool}" in 2026-02-22T22:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ enable_mirror cluster1 mirror_parent image1 journal 2026-02-22T22:34:54.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mode=journal 2026-02-22T22:34:54.491 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster=cluster1 mirror image enable mirror_parent/image1 journal 2026-02-22T22:34:54.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:Mirroring enabled 2026-02-22T22:34:54.521 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster=cluster1 info mirror_parent/image1 2026-02-22T22:34:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:rbd image 'image1': 2026-02-22T22:34:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: size 128 MiB in 32 objects 2026-02-22T22:34:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: order 22 (4 MiB objects) 2026-02-22T22:34:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: snapshot_count: 0 2026-02-22T22:34:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: id: 10f964608a1f 2026-02-22T22:34:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: block_name_prefix: rbd_data.10f964608a1f 2026-02-22T22:34:54.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: format: 2 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: op_features: 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: flags: 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: create_timestamp: Sun Feb 22 22:34:54 2026 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: access_timestamp: Sun Feb 22 22:34:54 2026 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: modify_timestamp: Sun Feb 22 22:34:54 2026 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: journal: 10f964608a1f 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: mirroring state: enabled 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: mirroring mode: journal 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: mirroring global id: a7f20be6-a3a8-406b-a451-ebaf7f117f7b 2026-02-22T22:34:54.548 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: mirroring primary: true 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent image2 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mode=journal 2026-02-22T22:34:54.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image cluster2 mirror_parent image2 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size=128 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/image2 2026-02-22T22:34:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ is_pool_mirror_mode_image mirror_parent 2026-02-22T22:34:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' pool = image ']' 2026-02-22T22:34:54.580 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ case "${pool}" in 2026-02-22T22:34:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ enable_mirror cluster2 mirror_parent image2 journal 2026-02-22T22:34:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:34:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mode=journal 2026-02-22T22:34:54.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster=cluster2 mirror image enable mirror_parent/image2 journal 2026-02-22T22:34:54.606 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:Mirroring enabled 2026-02-22T22:34:54.608 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster=cluster2 info mirror_parent/image2 2026-02-22T22:34:54.632 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:rbd image 'image2': 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: size 128 MiB in 32 objects 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: order 22 (4 MiB objects) 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: snapshot_count: 0 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: id: 10c6543f077d 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: block_name_prefix: rbd_data.10c6543f077d 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: format: 2 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: features: layering, exclusive-lock, journaling 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: op_features: 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: flags: 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: create_timestamp: Sun Feb 22 22:34:54 2026 2026-02-22T22:34:54.633 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: access_timestamp: Sun Feb 22 22:34:54 2026 2026-02-22T22:34:54.634 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: modify_timestamp: Sun Feb 22 22:34:54 2026 2026-02-22T22:34:54.634 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: journal: 10c6543f077d 2026-02-22T22:34:54.634 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: mirroring state: enabled 2026-02-22T22:34:54.634 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: mirroring mode: journal 2026-02-22T22:34:54.634 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: mirroring global id: 98914939-7176-4f23-830d-50bb026e9a66 2026-02-22T22:34:54.634 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: mirroring primary: true 2026-02-22T22:34:54.635 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image_and_enable_mirror cluster1 mirror_parent/ns1 image1 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mode=journal 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image cluster1 mirror_parent/ns1 image1 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:54.636 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:34:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size=128 2026-02-22T22:34:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.637 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/ns1/image1 2026-02-22T22:34:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ is_pool_mirror_mode_image mirror_parent/ns1 2026-02-22T22:34:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:34:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' pool = image ']' 2026-02-22T22:34:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ case "${pool}" in 2026-02-22T22:34:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 1 2026-02-22T22:34:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image_and_enable_mirror cluster2 mirror_parent/ns1 image2 2026-02-22T22:34:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:54.669 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local mode=journal 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ create_image cluster2 mirror_parent/ns1 image2 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.670 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:34:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:34:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size=128 2026-02-22T22:34:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:34:54.671 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 create --size 128 --image-feature layering,exclusive-lock,journaling mirror_parent/ns1/image2 2026-02-22T22:34:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ is_pool_mirror_mode_image mirror_parent/ns1 2026-02-22T22:34:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:34:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' pool = image ']' 2026-02-22T22:34:55.107 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ case "${pool}" in 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 1 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_started cluster2 mirror_parent image1 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_state cluster2 mirror_parent image1 started 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=started 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:55.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:34:56.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_image_replay_state cluster2 mirror_parent image1 started 2026-02-22T22:34:56.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:56.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:56.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:56.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local test_state=started 2026-02-22T22:34:56.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status_result 2026-02-22T22:34:56.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local current_state=stopped 2026-02-22T22:34:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemons cluster2 rbd mirror status mirror_parent/image1 2026-02-22T22:34:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_instance=cluster2 2026-02-22T22:34:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -i state 2026-02-22T22:34:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance=cluster2 2026-02-22T22:34:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local loop_instance 2026-02-22T22:34:56.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 0 2026-02-22T22:34:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:56.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:34:56.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror_parent/image1 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:56.114 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:56.115 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:34:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:34:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:34:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:56.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:34:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster2 2026-02-22T22:34:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:34:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:34:56.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:56.125 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:56.126 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror_parent/image1 2026-02-22T22:34:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:Invalid command: unused arguments: ['mirror_parent/image1'] 2026-02-22T22:34:56.167 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:34:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:admin_socket: invalid command 2026-02-22T22:34:56.168 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:56.169 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 1 2026-02-22T22:34:57.169 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:57.170 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:34:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror_parent/image1 2026-02-22T22:34:57.173 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:34:57.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:57.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:34:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster2 2026-02-22T22:34:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:34:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:34:57.176 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:57.185 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:57.186 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror_parent/image1 2026-02-22T22:34:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:Invalid command: unused arguments: ['mirror_parent/image1'] 2026-02-22T22:34:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:rbd mirror status : get status for rbd mirror 2026-02-22T22:34:57.228 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:admin_socket: invalid command 2026-02-22T22:34:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:57.229 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 2 2026-02-22T22:34:59.230 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:59.231 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:34:59.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:59.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror_parent/image1 2026-02-22T22:34:59.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:34:59.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:34:59.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:59.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:59.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:59.234 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:34:59.235 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster2 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:34:59.236 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:34:59.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.246 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.247 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror_parent/image1 2026-02-22T22:34:59.295 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ return 0 2026-02-22T22:34:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:34:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:34:59.296 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep -i Replaying 2026-02-22T22:34:59.297 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: "state": "Replaying" 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ current_state=started 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test started = started 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ write_image cluster1 mirror_parent image1 100 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local count=100 2026-02-22T22:34:59.298 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size= 2026-02-22T22:34:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:34:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ size=4096 2026-02-22T22:34:59.299 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 bench mirror_parent/image1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:34:59.333 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:34:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:34:59.345 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:elapsed: 0 ops: 100 ops/sec: 8333.46 bytes/sec: 33 MiB/s 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror_parent image1 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster2 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror_parent image1 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster2 2026-02-22T22:34:59.353 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:34:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:34:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:34:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:34:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:34:59.354 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:34:59.554 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster2 mirror_parent image1 2026-02-22T22:34:59.554 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:59.554 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:59.554 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image1 ']' 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/image1' 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster2 rbd mirror flush mirror_parent/image1 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster2 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster2 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:59.555 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:34:59.556 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:34:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:59.558 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror_parent/image1 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:34:59.559 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:59.560 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:34:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:34:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:59.561 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror_parent/image1 2026-02-22T22:34:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster1 mirror_parent image1 2026-02-22T22:34:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:59.620 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror_parent image1 '' 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror_parent/image1 2026-02-22T22:34:59.622 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:59.653 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:59.654 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=fdd6646f-222d-4bba-9490-3c5c8572882f, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:34:59.655 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.656 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster1 mirror_parent image1 2026-02-22T22:34:59.657 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror_parent image1 '..*' 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:59.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror_parent/image1 2026-02-22T22:34:59.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:59.691 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:59.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=fdd6646f-222d-4bba-9490-3c5c8572882f, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:34:59.693 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:34:59.695 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:59.696 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_tag=2 2026-02-22T22:34:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:34:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:59.697 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_tag=1 2026-02-22T22:34:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:34:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:34:59.698 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:59.699 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_entry=101 2026-02-22T22:34:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:34:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:34:59.700 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:34:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_entry=3 2026-02-22T22:34:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:34:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:34:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:34:59.701 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster2 mirror_parent image1 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image1 ']' 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/image1' 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster2 rbd mirror flush mirror_parent/image1 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster2 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster2 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:34:59.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:34:59.903 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror_parent/image1 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:34:59.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:34:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:34:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:34:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:34:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:34:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:34:59.905 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:34:59.916 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.916 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.917 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:34:59.917 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror_parent/image1 2026-02-22T22:34:59.965 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster1 mirror_parent image1 2026-02-22T22:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror_parent image1 '' 2026-02-22T22:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:34:59.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:34:59.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror_parent/image1 2026-02-22T22:34:59.968 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:59.998 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=fdd6646f-222d-4bba-9490-3c5c8572882f, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:34:59.999 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:35:00.000 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:00.001 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster1 mirror_parent image1 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror_parent image1 '..*' 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent-image1.status 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent-image1.status 2026-02-22T22:35:00.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:35:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror_parent/image1 2026-02-22T22:35:00.003 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:35:00.032 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:00.033 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=fdd6646f-222d-4bba-9490-3c5c8572882f, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:00.034 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image1.status 2026-02-22T22:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:00.036 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror_parent image1 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_status_in_pool_dir cluster2 mirror_parent image1 up+replaying primary_position 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:35:00.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:00.038 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_status_in_pool_dir cluster2 mirror_parent image1 up+replaying primary_position '' 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:01.037 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror image status mirror_parent/image1 2026-02-22T22:35:01.067 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='image1: 2026-02-22T22:35:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: global_id: a7f20be6-a3a8-406b-a451-ebaf7f117f7b 2026-02-22T22:35:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+replaying 2026-02-22T22:35:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:35:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: service: mirror.0 on trial198 2026-02-22T22:35:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:34:57 2026-02-22T22:35:01.068 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: peer_sites: 2026-02-22T22:35:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: name: cluster1 2026-02-22T22:35:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+stopped 2026-02-22T22:35:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: local image is primary 2026-02-22T22:35:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:35:00' 2026-02-22T22:35:01.069 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:35:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: state: up+replaying 2026-02-22T22:35:01.070 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:35:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:35:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:35:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:35:01.071 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^up+' 2026-02-22T22:35:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:35:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:35:01.072 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: service: mirror.0 on trial198 2026-02-22T22:35:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update 2026-02-22T22:35:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:35:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:57' 2026-02-22T22:35:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_mirror_pool_status_verbose cluster2 mirror_parent image1 up+replaying '2026-02-22 22:34:57' 2026-02-22T22:35:01.074 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'prev_last_update=2026-02-22 22:34:57' 2026-02-22T22:35:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:01.075 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format xml 2026-02-22T22:35:01.104 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='OKOKOK242594270mirror.0trial19819.2.3-689-gf90f4eaetrueOKimage1a7f20be6-a3a8-406b-a451-ebaf7f117f7bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42594270mirror.0trial1982026-02-22 22:34:57cluster1e2a1d4cc-f33f-4e97-a188-0c9d061c3151up+stoppedlocal image is primary2026-02-22 22:35:00image298914939-7176-4f23-830d-50bb026e9a66up+stoppedlocal image is primary42594270mirror.0trial1982026-02-22 22:35:00cluster1e2a1d4cc-f33f-4e97-a188-0c9d061c3151up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:34:57' 2026-02-22T22:35:01.104 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update state 2026-02-22T22:35:01.105 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/last_update' 2026-02-22T22:35:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:57' 2026-02-22T22:35:01.106 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/state' 2026-02-22T22:35:01.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ state=up+replaying 2026-02-22T22:35:01.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:35:01.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep up+replaying 2026-02-22T22:35:01.109 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:35:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_started cluster1 mirror_parent image2 2026-02-22T22:35:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:01.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_state cluster1 mirror_parent image2 started 2026-02-22T22:35:01.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:01.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:01.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:01.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=started 2026-02-22T22:35:01.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s 2026-02-22T22:35:01.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:01.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_image_replay_state cluster1 mirror_parent image2 started 2026-02-22T22:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local test_state=started 2026-02-22T22:35:02.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status_result 2026-02-22T22:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local current_state=stopped 2026-02-22T22:35:02.112 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/image2 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -i state 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_instance=cluster1 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance=cluster1 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local loop_instance 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 0 2026-02-22T22:35:02.113 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:02.116 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/image2 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1:0 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster1 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster1 2026-02-22T22:35:02.117 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster1 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster1:0 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster1 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:02.118 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:02.119 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:35:02.119 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster1 2026-02-22T22:35:02.119 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:35:02.119 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:35:02.119 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.129 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/image2 2026-02-22T22:35:02.174 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ return 0 2026-02-22T22:35:02.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:35:02.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:35:02.175 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep -i Replaying 2026-02-22T22:35:02.176 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: "state": "Replaying" 2026-02-22T22:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ current_state=started 2026-02-22T22:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test started = started 2026-02-22T22:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ write_image cluster2 mirror_parent image2 100 2026-02-22T22:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:02.177 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local count=100 2026-02-22T22:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size= 2026-02-22T22:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ size=4096 2026-02-22T22:35:02.178 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 bench mirror_parent/image2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:35:02.210 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:35:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:35:02.229 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:elapsed: 0 ops: 100 ops/sec: 5263.24 bytes/sec: 21 MiB/s 2026-02-22T22:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent image2 2026-02-22T22:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster1 2026-02-22T22:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror_parent image2 2026-02-22T22:35:02.237 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster1 2026-02-22T22:35:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:35:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:35:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:35:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:35:02.238 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:35:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster1 mirror_parent image2 2026-02-22T22:35:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:02.438 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image2 ']' 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/image2' 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster1 rbd mirror flush mirror_parent/image2 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster1 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster1 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:02.439 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:02.442 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror_parent/image2 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1:0 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster1:0 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:35:02.443 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster1 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster1 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:02.444 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster1 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster1 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:02.445 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:02.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror_parent/image2 2026-02-22T22:35:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster2 mirror_parent image2 2026-02-22T22:35:02.502 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent image2 '' 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.503 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/image2 2026-02-22T22:35:02.504 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.530 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:02.531 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:02.531 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:02.531 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:02.531 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:35:02.532 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.533 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster2 mirror_parent image2 2026-02-22T22:35:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:02.534 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent image2 '..*' 2026-02-22T22:35:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:35:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.535 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/image2 2026-02-22T22:35:02.536 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.561 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:02.562 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:35:02.563 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:35:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:35:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:35:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:02.565 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:35:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:02.566 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_tag=2 2026-02-22T22:35:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:35:02.567 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_tag=1 2026-02-22T22:35:02.568 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:02.569 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:35:02.569 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:02.569 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_entry=101 2026-02-22T22:35:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:35:02.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_entry=3 2026-02-22T22:35:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:35:02.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:35:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:35:02.572 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster1 mirror_parent image2 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image2 ']' 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/image2' 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster1 rbd mirror flush mirror_parent/image2 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster1 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:02.772 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster1 2026-02-22T22:35:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:35:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:35:02.773 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror_parent/image2 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1:0 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster1:0 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster1 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster1 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:35:02.774 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster1 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster1 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:02.775 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:02.786 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.786 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.786 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:02.786 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror_parent/image2 2026-02-22T22:35:02.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:02.835 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster2 mirror_parent image2 2026-02-22T22:35:02.835 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent image2 '' 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:02.837 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.837 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/image2 2026-02-22T22:35:02.837 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:02.865 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:02.866 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.867 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:02.870 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster2 mirror_parent image2 2026-02-22T22:35:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent image2 '..*' 2026-02-22T22:35:02.871 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:35:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.872 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/image2 2026-02-22T22:35:02.873 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:02.897 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:02.898 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:02.899 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent-image2.status 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent image2 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:35:02.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent image2 up+replaying primary_position 2026-02-22T22:35:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:35:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:35:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:02.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_status_in_pool_dir cluster1 mirror_parent image2 up+replaying primary_position '' 2026-02-22T22:35:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:03.902 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:35:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:35:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:03.903 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/image2 2026-02-22T22:35:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='image2: 2026-02-22T22:35:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: global_id: 98914939-7176-4f23-830d-50bb026e9a66 2026-02-22T22:35:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+replaying 2026-02-22T22:35:03.932 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:35:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: service: mirror.0 on trial198 2026-02-22T22:35:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:34:57 2026-02-22T22:35:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: peer_sites: 2026-02-22T22:35:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: name: cluster2 2026-02-22T22:35:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+stopped 2026-02-22T22:35:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: local image is primary 2026-02-22T22:35:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:35:00' 2026-02-22T22:35:03.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:35:03.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: state: up+replaying 2026-02-22T22:35:03.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:35:03.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:35:03.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:35:03.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:35:03.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^up+' 2026-02-22T22:35:03.936 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:35:03.936 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:35:03.936 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: service: mirror.0 on trial198 2026-02-22T22:35:03.937 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update 2026-02-22T22:35:03.937 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:35:03.938 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:57' 2026-02-22T22:35:03.938 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent image2 up+replaying '2026-02-22 22:34:57' 2026-02-22T22:35:03.938 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:03.938 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'prev_last_update=2026-02-22 22:34:57' 2026-02-22T22:35:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:03.939 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format xml 2026-02-22T22:35:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='OKOKOK242704272mirror.0trial19819.2.3-689-gf90f4eaetrueOKimage1a7f20be6-a3a8-406b-a451-ebaf7f117f7bup+stoppedlocal image is primary42704272mirror.0trial1982026-02-22 22:35:00cluster2fdd6646f-222d-4bba-9490-3c5c8572882fup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:34:57image298914939-7176-4f23-830d-50bb026e9a66up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42704272mirror.0trial1982026-02-22 22:34:57cluster2fdd6646f-222d-4bba-9490-3c5c8572882fup+stoppedlocal image is primary2026-02-22 22:35:00' 2026-02-22T22:35:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update state 2026-02-22T22:35:03.968 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image2'\'']/last_update' 2026-02-22T22:35:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:57' 2026-02-22T22:35:03.970 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image2'\'']/state' 2026-02-22T22:35:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ state=up+replaying 2026-02-22T22:35:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:35:03.972 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep up+replaying 2026-02-22T22:35:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:35:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:03.973 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster1 mirror_parent 2026-02-22T22:35:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:03.974 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:04.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4270","instance_id":"4272","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:00","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:57"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:57","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:00"}]}]}' 2026-02-22T22:35:04.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.states == {"replaying": 2}' 2026-02-22T22:35:04.006 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:04.007 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:04.007 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:04.007 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:04.007 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:04.007 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:04.039 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:57","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:00"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:00","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:57"}]}]}' 2026-02-22T22:35:04.039 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.states == {"replaying": 2}' 2026-02-22T22:35:04.040 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_started cluster2 mirror_parent/ns1 image1 2026-02-22T22:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:04.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_state cluster2 mirror_parent/ns1 image1 started 2026-02-22T22:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=started 2026-02-22T22:35:04.043 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s 2026-02-22T22:35:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:04.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_image_replay_state cluster2 mirror_parent/ns1 image1 started 2026-02-22T22:35:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:05.042 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local test_state=started 2026-02-22T22:35:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status_result 2026-02-22T22:35:05.043 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local current_state=stopped 2026-02-22T22:35:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemons cluster2 rbd mirror status mirror_parent/ns1/image1 2026-02-22T22:35:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_instance=cluster2 2026-02-22T22:35:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:35:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -i state 2026-02-22T22:35:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance=cluster2 2026-02-22T22:35:05.044 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local loop_instance 2026-02-22T22:35:05.045 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:05.045 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 0 2026-02-22T22:35:05.045 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:35:05.045 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:35:05.047 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster2:0 rbd mirror status mirror_parent/ns1/image1 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:05.048 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster2:0 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:35:05.049 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:35:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:35:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster2 2026-02-22T22:35:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster2 2026-02-22T22:35:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:35:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:35:05.050 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.060 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror status mirror_parent/ns1/image1 2026-02-22T22:35:05.108 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ return 0 2026-02-22T22:35:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:35:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:35:05.109 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep -i Replaying 2026-02-22T22:35:05.110 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: "state": "Replaying" 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ current_state=started 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test started = started 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ write_image cluster1 mirror_parent/ns1 image1 100 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local count=100 2026-02-22T22:35:05.111 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size= 2026-02-22T22:35:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:35:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ size=4096 2026-02-22T22:35:05.112 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster1 bench mirror_parent/ns1/image1 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:35:05.346 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:35:05.360 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:35:05.360 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:elapsed: 0 ops: 100 ops/sec: 7142.97 bytes/sec: 28 MiB/s 2026-02-22T22:35:05.367 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replay_complete cluster2 cluster1 mirror_parent/ns1 image1 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster2 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_journal_replay_complete cluster2 cluster1 mirror_parent/ns1 image1 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster2 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:05.368 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:35:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:35:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:35:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:35:05.369 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:35:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster2 mirror_parent/ns1 image1 2026-02-22T22:35:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:35:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image1 ']' 2026-02-22T22:35:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/ns1/image1' 2026-02-22T22:35:05.570 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster2 rbd mirror flush mirror_parent/ns1/image1 2026-02-22T22:35:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster2 2026-02-22T22:35:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster2 2026-02-22T22:35:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:35:05.571 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:35:05.572 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:05.575 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:05.575 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror_parent/ns1/image1 2026-02-22T22:35:05.575 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:35:05.575 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:35:05.576 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:05.577 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:35:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:35:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:05.578 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:35:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror_parent/ns1/image1 2026-02-22T22:35:05.638 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:05.638 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster1 mirror_parent/ns1 image1 2026-02-22T22:35:05.638 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:05.638 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror_parent/ns1 image1 '' 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:35:05.639 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror_parent/ns1/image1 2026-02-22T22:35:05.640 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:05.671 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:05.672 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=fdd6646f-222d-4bba-9490-3c5c8572882f, commit_position=[positions=[[object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3]]], state=connected] 2026-02-22T22:35:05.673 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:05.674 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster1 mirror_parent/ns1 image1 2026-02-22T22:35:05.675 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror_parent/ns1 image1 '..*' 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:35:05.676 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror_parent/ns1/image1 2026-02-22T22:35:05.677 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:05.708 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=fdd6646f-222d-4bba-9490-3c5c8572882f, commit_position=[positions=[[object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1], [object_number=0, tag_tid=2, entry_tid=0], [object_number=3, tag_tid=1, entry_tid=3]]], state=connected] 2026-02-22T22:35:05.709 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=2, tag_tid=2, entry_tid=2' 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=2, tag_tid=2, entry_tid=2' 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=2, tag_tid=2, entry_tid=2' '!=' '' 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=2, tag_tid=2, entry_tid=2' = '' 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_mirror_pos='object_number=2, tag_tid=2, entry_tid=2' 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:35:05.712 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_tag=2 2026-02-22T22:35:05.713 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=2, tag_tid=2, entry_tid=2' 2026-02-22T22:35:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:35:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:05.714 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_tag=2 2026-02-22T22:35:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:35:05.715 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:05.716 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_entry=101 2026-02-22T22:35:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=2, tag_tid=2, entry_tid=2' 2026-02-22T22:35:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:35:05.717 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_entry=2 2026-02-22T22:35:05.718 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 2 = 2 -a 101 -le 2 2026-02-22T22:35:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:35:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:35:05.719 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:35:05.920 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster2 mirror_parent/ns1 image1 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image1 ']' 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/ns1/image1' 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster2 rbd mirror flush mirror_parent/ns1/image1 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster2 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:05.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster2 2026-02-22T22:35:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:35:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster2 '!=' cluster2 ']' 2026-02-22T22:35:05.922 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster2:0 rbd mirror flush mirror_parent/ns1/image1 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster2:0 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:35:05.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster2 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster2 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:35:05.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:05.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:35:05.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:05.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok rbd mirror flush mirror_parent/ns1/image1 2026-02-22T22:35:05.984 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:05.985 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster1 mirror_parent/ns1 image1 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror_parent/ns1 image1 '' 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:35:05.986 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:35:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror_parent/ns1/image1 2026-02-22T22:35:05.987 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:06.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:06.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:06.018 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:06.018 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:06.018 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=fdd6646f-222d-4bba-9490-3c5c8572882f, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:06.019 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:06.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster1 mirror_parent/ns1 image1 2026-02-22T22:35:06.021 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster1 mirror_parent/ns1 image1 '..*' 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image1 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image1.status 2026-02-22T22:35:06.022 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 journal status --image mirror_parent/ns1/image1 2026-02-22T22:35:06.023 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:06.053 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:06.054 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=fdd6646f-222d-4bba-9490-3c5c8572882f, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:06.055 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image1.status 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replaying_status_in_pool_dir cluster2 mirror_parent/ns1 image1 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:35:06.057 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_status_in_pool_dir cluster2 mirror_parent/ns1 image1 up+replaying primary_position 2026-02-22T22:35:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:35:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:35:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:06.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_status_in_pool_dir cluster2 mirror_parent/ns1 image1 up+replaying primary_position '' 2026-02-22T22:35:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:35:07.058 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:35:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:07.059 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror image status mirror_parent/ns1/image1 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='image1: 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: global_id: c6dbc974-2640-48c8-a689-410a4f56e8cb 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+replaying 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: service: mirror.0 on trial198 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:34:57 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: peer_sites: 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: name: cluster1 2026-02-22T22:35:07.089 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+stopped 2026-02-22T22:35:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: local image is primary 2026-02-22T22:35:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:35:05' 2026-02-22T22:35:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:35:07.090 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: state: up+replaying 2026-02-22T22:35:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:35:07.091 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:35:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:35:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:35:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^up+' 2026-02-22T22:35:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:35:07.092 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:35:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: service: mirror.0 on trial198 2026-02-22T22:35:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update 2026-02-22T22:35:07.093 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:35:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:57' 2026-02-22T22:35:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_mirror_pool_status_verbose cluster2 mirror_parent/ns1 image1 up+replaying '2026-02-22 22:34:57' 2026-02-22T22:35:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image1 2026-02-22T22:35:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'prev_last_update=2026-02-22 22:34:57' 2026-02-22T22:35:07.095 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:07.096 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent/ns1 --verbose --format xml 2026-02-22T22:35:07.125 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='OKOKOK242594270mirror.0trial19819.2.3-689-gf90f4eaetrueOKimage1c6dbc974-2640-48c8-a689-410a4f56e8cbup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42594270mirror.0trial1982026-02-22 22:34:57cluster1e2a1d4cc-f33f-4e97-a188-0c9d061c3151up+stoppedlocal image is primary2026-02-22 22:35:05image2709307cf-569d-4149-b6d3-75da4225e68bup+stoppedlocal image is primary42594270mirror.0trial1982026-02-22 22:35:05cluster1e2a1d4cc-f33f-4e97-a188-0c9d061c3151up+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:34:58' 2026-02-22T22:35:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update state 2026-02-22T22:35:07.126 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/last_update' 2026-02-22T22:35:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:57' 2026-02-22T22:35:07.127 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image1'\'']/state' 2026-02-22T22:35:07.129 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ state=up+replaying 2026-02-22T22:35:07.129 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:35:07.129 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep up+replaying 2026-02-22T22:35:07.130 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:35:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:07.131 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_started cluster1 mirror_parent/ns1 image2 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_image_replay_state cluster1 mirror_parent/ns1 image2 started 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=started 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:07.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:08.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_image_replay_state cluster1 mirror_parent/ns1 image2 started 2026-02-22T22:35:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local test_state=started 2026-02-22T22:35:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status_result 2026-02-22T22:35:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local current_state=stopped 2026-02-22T22:35:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemons cluster1 rbd mirror status mirror_parent/ns1/image2 2026-02-22T22:35:08.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_instance=cluster1 2026-02-22T22:35:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:35:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance=cluster1 2026-02-22T22:35:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local loop_instance 2026-02-22T22:35:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sleep 0 2026-02-22T22:35:08.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -i state 2026-02-22T22:35:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:35:08.137 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ seq 0 0 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ admin_daemon cluster1:0 rbd mirror status mirror_parent/ns1/image2 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1:0 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ shift 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:08.138 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster1 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster1 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local local_cluster=cluster1:0 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster1 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:08.139 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster1:0 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=local_cluster 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster1 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval local_cluster=cluster1 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ local_cluster=cluster1 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:35:08.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.149 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror status mirror_parent/ns1/image2 2026-02-22T22:35:08.196 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ return 0 2026-02-22T22:35:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status_result=' "state": "Replaying"' 2026-02-22T22:35:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo ' "state": "Replaying"' 2026-02-22T22:35:08.197 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep -i Replaying 2026-02-22T22:35:08.198 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: "state": "Replaying" 2026-02-22T22:35:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ current_state=started 2026-02-22T22:35:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test started = started 2026-02-22T22:35:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ write_image cluster2 mirror_parent/ns1 image2 100 2026-02-22T22:35:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:08.199 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local count=100 2026-02-22T22:35:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local size= 2026-02-22T22:35:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:35:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ size=4096 2026-02-22T22:35:08.200 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 bench mirror_parent/ns1/image2 --io-type write --io-size 4096 --io-threads 1 --io-total 409600 --io-pattern rand 2026-02-22T22:35:08.231 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:bench type write io_size 4096 io_threads 1 bytes 409600 pattern random 2026-02-22T22:35:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: SEC OPS OPS/SEC BYTES/SEC 2026-02-22T22:35:08.245 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:elapsed: 0 ops: 100 ops/sec: 7142.97 bytes/sec: 28 MiB/s 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replay_complete cluster1 cluster2 mirror_parent/ns1 image2 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster1 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_journal_replay_complete cluster1 cluster2 mirror_parent/ns1 image2 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local local_cluster=cluster1 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:08.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s master_pos mirror_pos last_mirror_pos 2026-02-22T22:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local master_tag master_entry mirror_tag mirror_entry 2026-02-22T22:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:35:08.252 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:35:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster1 mirror_parent/ns1 image2 2026-02-22T22:35:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:35:08.454 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image2 ']' 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/ns1/image2' 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster1 rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster1 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster1 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:08.455 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:35:08.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:08.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1:0 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster1:0 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster1 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:08.459 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster1 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:08.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster1 2026-02-22T22:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster1 2026-02-22T22:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:08.461 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:08.470 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.471 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:08.517 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster2 mirror_parent/ns1 image2 2026-02-22T22:35:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:08.518 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent/ns1 image2 '' 2026-02-22T22:35:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:35:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:08.519 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:08.520 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.520 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/ns1/image2 2026-02-22T22:35:08.520 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:08.546 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:35:08.547 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.549 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster2 mirror_parent/ns1 image2 2026-02-22T22:35:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:08.550 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent/ns1 image2 '..*' 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:08.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/ns1/image2 2026-02-22T22:35:08.552 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:08.578 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=3, tag_tid=1, entry_tid=3], [object_number=2, tag_tid=1, entry_tid=2], [object_number=1, tag_tid=1, entry_tid=1], [object_number=0, tag_tid=1, entry_tid=0]]], state=connected] 2026-02-22T22:35:08.579 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' '!=' '' 2026-02-22T22:35:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:35:08.581 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=3, tag_tid=1, entry_tid=3' = '' 2026-02-22T22:35:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_mirror_pos='object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:35:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:08.582 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_tag=2 2026-02-22T22:35:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:35:08.583 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:08.584 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_tag=1 2026-02-22T22:35:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:35:08.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:08.586 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_entry=101 2026-02-22T22:35:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:35:08.587 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:08.588 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_entry=3 2026-02-22T22:35:08.588 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 2 = 1 -a 101 -le 3 2026-02-22T22:35:08.588 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:35:08.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:35:08.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:35:08.790 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster1 mirror_parent/ns1 image2 2026-02-22T22:35:08.790 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image2 ']' 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/ns1/image2' 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster1 rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster1 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster1 2026-02-22T22:35:08.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:35:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:08.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:35:08.796 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:08.796 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:08.796 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:08.796 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1:0 2026-02-22T22:35:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:35:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster1:0 2026-02-22T22:35:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:35:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:35:08.797 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster1 2026-02-22T22:35:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:35:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster1 2026-02-22T22:35:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:35:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:35:08.798 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:35:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:08.799 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster1 2026-02-22T22:35:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster1 2026-02-22T22:35:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:08.800 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:08.805 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.805 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.805 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:08.806 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster2 mirror_parent/ns1 image2 2026-02-22T22:35:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent/ns1 image2 '' 2026-02-22T22:35:08.855 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:35:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:08.856 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:08.857 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.857 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/ns1/image2 2026-02-22T22:35:08.857 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:08.884 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=0, tag_tid=2, entry_tid=4], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1]]], state=connected] 2026-02-22T22:35:08.885 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.886 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:08.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster2 mirror_parent/ns1 image2 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent/ns1 image2 '..*' 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:08.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/ns1/image2 2026-02-22T22:35:08.889 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:08.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:08.916 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:08.916 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=0, tag_tid=2, entry_tid=4], [object_number=3, tag_tid=2, entry_tid=3], [object_number=2, tag_tid=2, entry_tid=2], [object_number=1, tag_tid=2, entry_tid=1]]], state=connected] 2026-02-22T22:35:08.916 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=0, tag_tid=2, entry_tid=4' 2026-02-22T22:35:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=0, tag_tid=2, entry_tid=4' 2026-02-22T22:35:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=0, tag_tid=2, entry_tid=4' '!=' 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:08.918 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:35:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 'object_number=0, tag_tid=2, entry_tid=4' = 'object_number=3, tag_tid=1, entry_tid=3' 2026-02-22T22:35:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_mirror_pos='object_number=0, tag_tid=2, entry_tid=4' 2026-02-22T22:35:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:35:08.919 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_tag=2 2026-02-22T22:35:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=0, tag_tid=2, entry_tid=4' 2026-02-22T22:35:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'tag_tid=[0-9]*' 2026-02-22T22:35:08.920 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_tag=2 2026-02-22T22:35:08.921 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:08.922 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:35:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_entry=101 2026-02-22T22:35:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo 'object_number=0, tag_tid=2, entry_tid=4' 2026-02-22T22:35:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ grep -Eo 'entry_tid=[0-9]*' 2026-02-22T22:35:08.923 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cut -d= -f 2 2026-02-22T22:35:08.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_entry=4 2026-02-22T22:35:08.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test 2 = 2 -a 101 -le 4 2026-02-22T22:35:08.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ true 2026-02-22T22:35:08.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0.2 0.4 0.8 1.6 2 2 4 4 8 8 16 16 32 32 2026-02-22T22:35:08.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0.2 2026-02-22T22:35:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ flush cluster1 mirror_parent/ns1 image2 2026-02-22T22:35:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:09.127 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'cmd=rbd mirror flush' 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n image2 ']' 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cmd='rbd mirror flush mirror_parent/ns1/image2' 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemons cluster1 rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_instance=cluster1 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance=cluster1 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local loop_instance 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:09.128 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' cluster1 '!=' cluster1 ']' 2026-02-22T22:35:09.129 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:09.131 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for loop_instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ admin_daemon cluster1:0 rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1:0 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ shift 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local val=cluster1:0 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster_var_name=cluster 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local instance_var_name=instance 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cluster=cluster1 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ instance=0 2026-02-22T22:35:09.132 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval cluster=cluster1 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ eval instance=0 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:09.133 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster1 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster1 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:09.134 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local asok_file=/var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -S /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:09.140 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --admin-daemon /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok rbd mirror flush mirror_parent/ns1/image2 2026-02-22T22:35:09.187 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_master_journal_position cluster2 mirror_parent/ns1 image2 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent/ns1 image2 '' 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local id_regexp= 2026-02-22T22:35:09.188 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:09.189 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/ns1/image2 2026-02-22T22:35:09.190 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:09.216 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:09.217 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:09.218 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ master_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_mirror_journal_position cluster2 mirror_parent/ns1 image2 2026-02-22T22:35:09.219 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_journal_position cluster2 mirror_parent/ns1 image2 '..*' 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local image=image2 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local 'id_regexp=..*' 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ mkfname cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo cluster2-mirror_parent/ns1-image2.status 2026-02-22T22:35:09.220 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ sed -e 's|[/ ]|_|g' 2026-02-22T22:35:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local status_log=/tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 journal status --image mirror_parent/ns1/image2 2026-02-22T22:35:09.221 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ tee /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:minimum_set: 0 2026-02-22T22:35:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:active_set: 0 2026-02-22T22:35:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:registered clients: 2026-02-22T22:35:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:09.248 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: [id=e2a1d4cc-f33f-4e97-a188-0c9d061c3151, commit_position=[positions=[[object_number=1, tag_tid=2, entry_tid=101], [object_number=0, tag_tid=2, entry_tid=100], [object_number=3, tag_tid=2, entry_tid=99], [object_number=2, tag_tid=2, entry_tid=98]]], state=connected] 2026-02-22T22:35:09.249 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^.*\[id=..*,.*positions=\[\[([^]]*)\],.*state=connected.*$/\1/p' /tmp/tmp.O4E04S2FH2/cluster2-mirror_parent_ns1-image2.status 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ mirror_pos='object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n 'object_number=1, tag_tid=2, entry_tid=101' -a 'object_number=1, tag_tid=2, entry_tid=101' = 'object_number=1, tag_tid=2, entry_tid=101' 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_replaying_status_in_pool_dir cluster1 mirror_parent/ns1 image2 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' journal = journal ']' 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_status_in_pool_dir cluster1 mirror_parent/ns1 image2 up+replaying primary_position 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:09.251 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_status_in_pool_dir cluster1 mirror_parent/ns1 image2 up+replaying primary_position '' 2026-02-22T22:35:10.252 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local description_pattern=primary_position 2026-02-22T22:35:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local service_pattern= 2026-02-22T22:35:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:10.253 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror image status mirror_parent/ns1/image2 2026-02-22T22:35:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='image2: 2026-02-22T22:35:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: global_id: 709307cf-569d-4149-b6d3-75da4225e68b 2026-02-22T22:35:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+replaying 2026-02-22T22:35:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:35:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: service: mirror.0 on trial198 2026-02-22T22:35:10.282 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:34:58 2026-02-22T22:35:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: peer_sites: 2026-02-22T22:35:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: name: cluster2 2026-02-22T22:35:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: state: up+stopped 2026-02-22T22:35:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: description: local image is primary 2026-02-22T22:35:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr: last_update: 2026-02-22 22:35:05' 2026-02-22T22:35:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ state: .*up+replaying' 2026-02-22T22:35:10.283 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: state: up+replaying 2026-02-22T22:35:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^ description: .*primary_position' 2026-02-22T22:35:10.284 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: description: replaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}} 2026-02-22T22:35:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n '' ']' 2026-02-22T22:35:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:35:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep '^up+' 2026-02-22T22:35:10.285 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:35:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep 'service: *mirror..* on ' 2026-02-22T22:35:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout: service: mirror.0 on trial198 2026-02-22T22:35:10.286 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update 2026-02-22T22:35:10.287 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ sed -nEe 's/^ last_update: *(.*) *$/\1/p' 2026-02-22T22:35:10.288 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:58' 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_mirror_pool_status_verbose cluster1 mirror_parent/ns1 image2 up+replaying '2026-02-22 22:34:58' 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent/ns1 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local image=image2 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state_pattern=up+replaying 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local 'prev_last_update=2026-02-22 22:34:58' 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:10.289 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent/ns1 --verbose --format xml 2026-02-22T22:35:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='OKOKOK242704272mirror.0trial19819.2.3-689-gf90f4eaetrueOKimage1c6dbc974-2640-48c8-a689-410a4f56e8cbup+stoppedlocal image is primary42704272mirror.0trial1982026-02-22 22:35:10cluster2fdd6646f-222d-4bba-9490-3c5c8572882fup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}2026-02-22 22:34:57image2709307cf-569d-4149-b6d3-75da4225e68bup+replayingreplaying, {"bytes_per_second":0.0,"entries_behind_primary":3,"entries_per_second":0.0,"non_primary_position":{},"primary_position":{"entry_tid":3,"object_number":3,"tag_tid":1}}42704272mirror.0trial1982026-02-22 22:34:58cluster2fdd6646f-222d-4bba-9490-3c5c8572882fup+stoppedlocal image is primary2026-02-22 22:35:05' 2026-02-22T22:35:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local last_update state 2026-02-22T22:35:10.321 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image2'\'']/last_update' 2026-02-22T22:35:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ last_update='2026-02-22 22:34:58' 2026-02-22T22:35:10.322 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ xmlstarlet sel -t -v '//images/image[name='\''image2'\'']/state' 2026-02-22T22:35:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ state=up+replaying 2026-02-22T22:35:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo up+replaying 2026-02-22T22:35:10.324 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ grep up+replaying 2026-02-22T22:35:10.325 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:up+replaying 2026-02-22T22:35:10.326 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:10.326 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:10.326 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster1 mirror_parent/ns1 2026-02-22T22:35:10.326 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:10.326 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:10.326 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:10.326 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent/ns1 --verbose --format json 2026-02-22T22:35:10.357 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4270","instance_id":"4272","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"c6dbc974-2640-48c8-a689-410a4f56e8cb","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:10","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:57"}]},{"name":"image2","global_id":"709307cf-569d-4149-b6d3-75da4225e68b","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:58","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:10"}]}]}' 2026-02-22T22:35:10.357 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.states == {"replaying": 2}' 2026-02-22T22:35:10.358 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent/ns1 2026-02-22T22:35:10.359 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent/ns1 2026-02-22T22:35:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:10.360 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent/ns1 --verbose --format json 2026-02-22T22:35:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"c6dbc974-2640-48c8-a689-410a4f56e8cb","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:57","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:10"}]},{"name":"image2","global_id":"709307cf-569d-4149-b6d3-75da4225e68b","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:10","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:58"}]}]}' 2026-02-22T22:35:10.391 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.states == {"replaying": 2}' 2026-02-22T22:35:10.392 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ testlog 'TEST: pool replayer and callout cleanup when peer is updated' 2026-02-22T22:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ tee -a /tmp/tmp.O4E04S2FH2/rbd-mirror.test.log 2026-02-22T22:35:10.393 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ date '+%F %T' 2026-02-22T22:35:10.394 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo 2026-02-22 22:35:10 TEST: pool replayer and callout cleanup when peer is updated 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:2026-02-22 22:35:10 TEST: pool replayer and callout cleanup when peer is updated 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster1 mirror_parent OK 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=OK 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster1 mirror_parent 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:10.395 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:10.425 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4270","instance_id":"4272","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:10","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:57"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:57","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:10"}]}]}' 2026-02-22T22:35:10.425 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:35:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=OK 2026-02-22T22:35:10.427 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:10.428 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:57","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:10"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:10","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:57"}]}]}' 2026-02-22T22:35:10.458 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:10.460 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:57","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:10"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:10","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:57"}]}]}' 2026-02-22T22:35:10.490 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:10.491 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:10.492 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.daemon_health == "OK"' 2026-02-22T22:35:10.493 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:10.494 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.daemons[0].health == "OK"' 2026-02-22T22:35:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:10.495 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.daemons[0] | has("callouts") | not' 2026-02-22T22:35:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:10.497 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ jq -r '.daemons[0].service_id' 2026-02-22T22:35:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ OLD_SERVICE_ID=4259 2026-02-22T22:35:10.499 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ jq -r '.daemons[0].instance_id' 2026-02-22T22:35:10.501 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ OLD_INSTANCE_ID=4270 2026-02-22T22:35:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 --pool mirror_parent mirror pool info --format json 2026-02-22T22:35:10.502 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ jq -r '.peers[0].uuid' 2026-02-22T22:35:10.526 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ PEER_UUID=9a859c9c-dff4-4db4-9447-170ff6f39e7e 2026-02-22T22:35:10.526 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 --pool mirror_parent mirror pool peer set 9a859c9c-dff4-4db4-9447-170ff6f39e7e client client.invalid 2026-02-22T22:35:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_health_state cluster2 mirror_parent ERROR 2026-02-22T22:35:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:10.551 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=ERROR 2026-02-22T22:35:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s 2026-02-22T22:35:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:10.552 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent ERROR 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=ERROR 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:11.553 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:11.554 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:57","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:10"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:10","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:57"}]}]}' 2026-02-22T22:35:11.585 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:35:11.586 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:false 2026-02-22T22:35:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:11.587 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 2 2026-02-22T22:35:13.588 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent ERROR 2026-02-22T22:35:13.588 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:13.588 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=ERROR 2026-02-22T22:35:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:13.589 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:34:57","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:10"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4259","instance_id":"4270","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:10","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":3,\"entries_per_second\":0.0,\"non_primary_position\":{},\"primary_position\":{\"entry_tid\":3,\"object_number\":3,\"tag_tid\":1}}","last_update":"2026-02-22 22:34:57"}]}]}' 2026-02-22T22:35:13.620 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:35:13.621 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:false 2026-02-22T22:35:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:13.622 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 4 2026-02-22T22:35:17.624 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent ERROR 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=ERROR 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:17.625 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:15"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:17.655 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:35:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:false 2026-02-22T22:35:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:17.657 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 8 2026-02-22T22:35:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent ERROR 2026-02-22T22:35:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:25.658 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=ERROR 2026-02-22T22:35:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:25.659 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:25"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:25.690 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:35:25.691 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster1 mirror_parent WARNING 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=WARNING 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster1 mirror_parent 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:25.692 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"WARNING","daemon_health":"OK","image_health":"WARNING","states":{"unknown":2}},"daemons":[{"service_id":"4270","instance_id":"4272","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:25","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:35:15"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:25.724 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "WARNING"' 2026-02-22T22:35:25.725 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:25.726 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:25"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:25.757 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "ERROR"' 2026-02-22T22:35:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:25.758 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.daemon_health == "ERROR"' 2026-02-22T22:35:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:25.760 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.daemons[0].health == "ERROR"' 2026-02-22T22:35:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:25.762 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.daemons[0].callouts == ["unable to connect to remote cluster"]' 2026-02-22T22:35:25.763 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:25.764 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rbd --cluster cluster2 --pool mirror_parent mirror pool peer set 9a859c9c-dff4-4db4-9447-170ff6f39e7e client client.rbd-mirror-peer 2026-02-22T22:35:25.789 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ wait_for_health_state cluster2 mirror_parent OK 2026-02-22T22:35:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=OK 2026-02-22T22:35:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local s 2026-02-22T22:35:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:25.790 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 1 2026-02-22T22:35:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:35:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:26.791 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=OK 2026-02-22T22:35:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:26.792 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:25"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:26.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:false 2026-02-22T22:35:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:26.825 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 2 2026-02-22T22:35:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:35:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=OK 2026-02-22T22:35:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:28.827 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:28.828 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:28.859 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"down+stopped","description":"stopped","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:25"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"down+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:28.859 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:28.860 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:false 2026-02-22T22:35:28.861 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:28.861 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 4 2026-02-22T22:35:32.862 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=OK 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:32.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:32.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"ERROR","daemon_health":"ERROR","image_health":"OK","states":{"replaying":1,"stopped":1}},"daemons":[{"service_id":"4259","instance_id":"4270","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":false,"health":"ERROR","callouts":["unable to connect to remote cluster"]}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"down+unknown","description":"status not found","last_update":"","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:30"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"down+unknown","description":"status not found","last_update":"","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:32.896 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:false 2026-02-22T22:35:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 1 2 4 8 8 8 8 8 8 8 8 16 16 2026-02-22T22:35:32.897 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 8 2026-02-22T22:35:40.899 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster2 mirror_parent OK 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=OK 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:40.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4259","instance_id":"4425","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":0,\"entries_per_second\":0.0,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","daemon_service":{"service_id":"4259","instance_id":"4425","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:34","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:40"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4259","instance_id":"4425","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:40","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:40.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:40.933 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ return 0 2026-02-22T22:35:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test_health_state cluster1 mirror_parent OK 2026-02-22T22:35:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pool=mirror_parent 2026-02-22T22:35:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local state=OK 2026-02-22T22:35:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local status 2026-02-22T22:35:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster1 mirror_parent 2026-02-22T22:35:40.934 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:40.935 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster1 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ status='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4270","instance_id":"4272","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:40","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":0,\"entries_per_second\":0.0,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:34"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","daemon_service":{"service_id":"4270","instance_id":"4272","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:15","peer_sites":[{"site_name":"cluster2","mirror_uuids":"fdd6646f-222d-4bba-9490-3c5c8572882f","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:40"}]}]}' 2026-02-22T22:35:40.965 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:40.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ get_pool_status_json cluster2 mirror_parent 2026-02-22T22:35:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local pool=mirror_parent 2026-02-22T22:35:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ CEPH_ARGS= 2026-02-22T22:35:40.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ rbd --cluster cluster2 mirror pool status mirror_parent --verbose --format json 2026-02-22T22:35:40.997 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ POOL_STATUS='{"summary":{"health":"OK","daemon_health":"OK","image_health":"OK","states":{"replaying":2}},"daemons":[{"service_id":"4259","instance_id":"4425","client_id":"mirror.0","hostname":"trial198","ceph_version":"19.2.3-689-gf90f4eae","leader":true,"health":"OK"}],"images":[{"name":"image1","global_id":"a7f20be6-a3a8-406b-a451-ebaf7f117f7b","state":"up+replaying","description":"replaying, {\"bytes_per_second\":0.0,\"entries_behind_primary\":0,\"entries_per_second\":0.0,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","daemon_service":{"service_id":"4259","instance_id":"4425","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:34","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+stopped","description":"local image is primary","last_update":"2026-02-22 22:35:40"}]},{"name":"image2","global_id":"98914939-7176-4f23-830d-50bb026e9a66","state":"up+stopped","description":"local image is primary","daemon_service":{"service_id":"4259","instance_id":"4425","daemon_id":"mirror.0","hostname":"trial198"},"last_update":"2026-02-22 22:35:40","peer_sites":[{"site_name":"cluster1","mirror_uuids":"e2a1d4cc-f33f-4e97-a188-0c9d061c3151","state":"up+replaying","description":"replaying, {\"bytes_per_second\":41439.8,\"entries_behind_primary\":0,\"entries_per_second\":10.6,\"non_primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2},\"primary_position\":{\"entry_tid\":101,\"object_number\":1,\"tag_tid\":2}}","last_update":"2026-02-22 22:35:15"}]}]}' 2026-02-22T22:35:40.998 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.health == "OK"' 2026-02-22T22:35:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:40.999 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.summary.daemon_health == "OK"' 2026-02-22T22:35:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:41.001 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.daemons[0].health == "OK"' 2026-02-22T22:35:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:41.003 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ jq -e '.daemons[0] | has("callouts") | not' 2026-02-22T22:35:41.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:true 2026-02-22T22:35:41.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ jq -r '.daemons[0].service_id' 2026-02-22T22:35:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ NEW_SERVICE_ID=4259 2026-02-22T22:35:41.007 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ jq -r '.daemons[0].instance_id' 2026-02-22T22:35:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ NEW_INSTANCE_ID=4425 2026-02-22T22:35:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ (( OLD_SERVICE_ID == NEW_SERVICE_ID )) 2026-02-22T22:35:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ (( OLD_INSTANCE_ID < NEW_INSTANCE_ID )) 2026-02-22T22:35:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ cleanup 0 2026-02-22T22:35:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local error_code=0 2026-02-22T22:35:41.010 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ set +e 2026-02-22T22:35:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 -ne 0 ']' 2026-02-22T22:35:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -z '' ']' 2026-02-22T22:35:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster instance 2026-02-22T22:35:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:35:41.011 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --cluster cluster1 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:35:41.895 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:pool 'mirror' does not exist 2026-02-22T22:35:41.903 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:35:41.904 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --cluster cluster2 osd pool rm mirror mirror --yes-i-really-really-mean-it 2026-02-22T22:35:42.806 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:pool 'mirror' does not exist 2026-02-22T22:35:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:35:42.817 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --cluster cluster1 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:35:43.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:35:43.907 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ CEPH_ARGS= 2026-02-22T22:35:43.908 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ceph --cluster cluster2 osd pool rm mirror_parent mirror_parent --yes-i-really-really-mean-it 2026-02-22T22:35:44.810 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:pool 'mirror_parent' does not exist 2026-02-22T22:35:44.819 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:35:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ stop_mirrors cluster1 2026-02-22T22:35:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1 2026-02-22T22:35:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local sig= 2026-02-22T22:35:44.820 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ stop_mirror cluster1:0 '' 2026-02-22T22:35:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster1:0 2026-02-22T22:35:44.822 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local sig= 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pid 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_pid_file cluster1:0 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster1:0 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster1:0 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster1 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:44.823 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval cluster=cluster1 2026-02-22T22:35:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ cluster=cluster1 2026-02-22T22:35:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:35:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:35:44.824 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:35:44.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:35:44.834 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cat /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:35:44.835 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ pid=49024 2026-02-22T22:35:44.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n 49024 ']' 2026-02-22T22:35:44.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ kill 49024 2026-02-22T22:35:44.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:35:44.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:44.836 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ps auxww 2026-02-22T22:35:44.837 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ awk -v pid=49024 '$2 == pid {print; exit 1}' 2026-02-22T22:35:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:35:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ps auxww 2026-02-22T22:35:44.863 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ awk -v pid=49024 '$2 == pid {print; exit 1}' 2026-02-22T22:35:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster1:0 cluster1 2026-02-22T22:35:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:35:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:44.887 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster1 2026-02-22T22:35:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster1 2026-02-22T22:35:44.888 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:44.889 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:44.898 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster1:0 cluster2 2026-02-22T22:35:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster1:0 2026-02-22T22:35:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 local_cluster instance 2026-02-22T22:35:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:44.900 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster1 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster1 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:44.901 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'admin socket' 2026-02-22T22:35:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:44.912 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.asok 2026-02-22T22:35:44.914 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_pid_file cluster1:0 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1:0 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster1:0 cluster instance 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster1:0 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster1 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster1:0 ']' 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster1 2026-02-22T22:35:44.915 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster1 2026-02-22T22:35:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:44.916 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster1 --name client.mirror.0 'pid file' 2026-02-22T22:35:44.925 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:35:44.926 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster1-client.mirror.0.pid 2026-02-22T22:35:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for cluster in "${CLUSTER1}" "${CLUSTER2}" 2026-02-22T22:35:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ stop_mirrors cluster2 2026-02-22T22:35:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2 2026-02-22T22:35:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local sig= 2026-02-22T22:35:44.927 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ seq 0 0 2026-02-22T22:35:44.929 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for instance in `seq 0 ${LAST_MIRROR_INSTANCE}` 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ stop_mirror cluster2:0 '' 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local cluster=cluster2:0 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local sig= 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test -n '' 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ local pid 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ daemon_pid_file cluster2:0 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster=cluster2:0 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local val=cluster2:0 2026-02-22T22:35:44.930 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local cluster_var_name=cluster 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local instance_var_name=instance 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval cluster=cluster2 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ cluster=cluster2 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ eval instance=0 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ instance=0 2026-02-22T22:35:44.931 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:35:44.940 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:35:44.941 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cat /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:35:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ pid=49069 2026-02-22T22:35:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -n 49069 ']' 2026-02-22T22:35:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ kill 49069 2026-02-22T22:35:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ for s in 0 1 2 4 8 16 32 2026-02-22T22:35:44.942 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ sleep 0 2026-02-22T22:35:44.943 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ps auxww 2026-02-22T22:35:44.943 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ awk -v pid=49069 '$2 == pid {print; exit 1}' 2026-02-22T22:35:44.966 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ break 2026-02-22T22:35:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ ps auxww 2026-02-22T22:35:44.967 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ awk -v pid=49069 '$2 == pid {print; exit 1}' 2026-02-22T22:35:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster1 2026-02-22T22:35:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:35:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster1 2026-02-22T22:35:44.989 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:44.990 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:35:45.001 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:45.002 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_asok_file cluster2:0 cluster2 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local local_cluster=cluster2:0 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 local_cluster instance 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=local_cluster 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:35:45.004 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval local_cluster=cluster2 2026-02-22T22:35:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ local_cluster=cluster2 2026-02-22T22:35:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:45.005 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'admin socket' 2026-02-22T22:35:45.016 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:45.017 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.asok 2026-02-22T22:35:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ daemon_pid_file cluster2:0 2026-02-22T22:35:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster=cluster2:0 2026-02-22T22:35:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance 2026-02-22T22:35:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ set_cluster_instance cluster2:0 cluster instance 2026-02-22T22:35:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local val=cluster2:0 2026-02-22T22:35:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local cluster_var_name=cluster 2026-02-22T22:35:45.019 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ local instance_var_name=instance 2026-02-22T22:35:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ cluster=cluster2 2026-02-22T22:35:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ instance=0 2026-02-22T22:35:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ '[' 0 = cluster2:0 ']' 2026-02-22T22:35:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval cluster=cluster2 2026-02-22T22:35:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ cluster=cluster2 2026-02-22T22:35:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ eval instance=0 2026-02-22T22:35:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ instance=0 2026-02-22T22:35:45.020 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+++ ceph-conf --cluster cluster2 --name client.mirror.0 'pid file' 2026-02-22T22:35:45.030 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:++ echo /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:35:45.030 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rm -f /var/run/ceph/rbd-mirror.cluster2-client.mirror.0.pid 2026-02-22T22:35:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' -z 1 ']' 2026-02-22T22:35:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ test '' = /tmp/tmp.O4E04S2FH2 2026-02-22T22:35:45.031 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ rm -Rf /tmp/tmp.O4E04S2FH2 2026-02-22T22:35:45.032 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ '[' 0 -eq 0 ']' 2026-02-22T22:35:45.033 INFO:tasks.workunit.cluster1.client.mirror.trial198.stdout:OK 2026-02-22T22:35:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ echo OK 2026-02-22T22:35:45.034 INFO:tasks.workunit.cluster1.client.mirror.trial198.stderr:+ exit 0 2026-02-22T22:35:45.034 INFO:teuthology.orchestra.run:Running command with timeout 3600 2026-02-22T22:35:45.035 DEBUG:teuthology.orchestra.run.trial198:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror/tmp 2026-02-22T22:35:45.101 INFO:tasks.workunit:Stopping ['rbd/rbd_mirror_bootstrap.sh'] on cluster1.client.mirror... 2026-02-22T22:35:45.101 DEBUG:teuthology.orchestra.run.trial198:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.cluster1.client.mirror /home/ubuntu/cephtest/clone.cluster1.client.mirror 2026-02-22T22:35:45.439 DEBUG:teuthology.parallel:result is None 2026-02-22T22:35:45.439 DEBUG:teuthology.orchestra.run.trial198:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:35:45.464 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:35:45.464 DEBUG:teuthology.orchestra.run.trial198:> rmdir -- /home/ubuntu/cephtest/mnt.cluster1.mirror 2026-02-22T22:35:45.521 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.cluster1.mirror/client.mirror 2026-02-22T22:35:45.521 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:35:45.586 INFO:tasks.ceph.ceph_manager.cluster2:waiting for clean 2026-02-22T22:35:45.586 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:35:45.748 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:35:45.749 INFO:teuthology.orchestra.run.trial198.stderr:dumped all 2026-02-22T22:35:45.757 INFO:teuthology.orchestra.run.trial198.stdout:{"pg_ready":true,"pg_map":{"version":93,"stamp":"2026-02-22T22:35:44.759947+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":101416,"kb_used_data":8052,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283014104,"statfs":{"total":289910292480,"available":289806442496,"internally_reserved":0,"allocated":8245248,"data_stored":7589531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-566223189,"num_objects":-237,"num_object_clones":0,"num_object_copies":-474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-237,"num_whiteouts":0,"num_read":-4099,"num_read_kb":-4299,"num_write":-974,"num_write_kb":-3525,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.062275"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874578+0000","last_change":"2026-02-22T22:34:02.868487+0000","last_active":"2026-02-22T22:35:04.874578+0000","last_peered":"2026-02-22T22:35:04.874578+0000","last_clean":"2026-02-22T22:35:04.874578+0000","last_became_active":"2026-02-22T22:34:00.867294+0000","last_became_peered":"2026-02-22T22:34:00.867294+0000","last_unstale":"2026-02-22T22:35:04.874578+0000","last_undegraded":"2026-02-22T22:35:04.874578+0000","last_fullsized":"2026-02-22T22:35:04.874578+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:49.554408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094867900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874665+0000","last_change":"2026-02-22T22:34:02.868266+0000","last_active":"2026-02-22T22:35:04.874665+0000","last_peered":"2026-02-22T22:35:04.874665+0000","last_clean":"2026-02-22T22:35:04.874665+0000","last_became_active":"2026-02-22T22:34:00.867282+0000","last_became_peered":"2026-02-22T22:34:00.867282+0000","last_unstale":"2026-02-22T22:35:04.874665+0000","last_undegraded":"2026-02-22T22:35:04.874665+0000","last_fullsized":"2026-02-22T22:35:04.874665+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:32:35.739961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072328099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874572+0000","last_change":"2026-02-22T22:34:02.868262+0000","last_active":"2026-02-22T22:35:04.874572+0000","last_peered":"2026-02-22T22:35:04.874572+0000","last_clean":"2026-02-22T22:35:04.874572+0000","last_became_active":"2026-02-22T22:34:00.867317+0000","last_became_peered":"2026-02-22T22:34:00.867317+0000","last_unstale":"2026-02-22T22:35:04.874572+0000","last_undegraded":"2026-02-22T22:35:04.874572+0000","last_fullsized":"2026-02-22T22:35:04.874572+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:39:15.990906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075292700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.873588+0000","last_change":"2026-02-22T22:34:02.867946+0000","last_active":"2026-02-22T22:35:04.873588+0000","last_peered":"2026-02-22T22:35:04.873588+0000","last_clean":"2026-02-22T22:35:04.873588+0000","last_became_active":"2026-02-22T22:34:00.867325+0000","last_became_peered":"2026-02-22T22:34:00.867325+0000","last_unstale":"2026-02-22T22:35:04.873588+0000","last_undegraded":"2026-02-22T22:35:04.873588+0000","last_fullsized":"2026-02-22T22:35:04.873588+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:29:12.873846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043021699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":62,"reported_epoch":38,"state":"active+clean","last_fresh":"2026-02-22T22:35:42.764066+0000","last_change":"2026-02-22T22:34:02.869362+0000","last_active":"2026-02-22T22:35:42.764066+0000","last_peered":"2026-02-22T22:35:42.764066+0000","last_clean":"2026-02-22T22:35:42.764066+0000","last_became_active":"2026-02-22T22:34:00.867085+0000","last_became_peered":"2026-02-22T22:34:00.867085+0000","last_unstale":"2026-02-22T22:35:42.764066+0000","last_undegraded":"2026-02-22T22:35:42.764066+0000","last_fullsized":"2026-02-22T22:35:42.764066+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:00:12.510734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089501999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.845344+0000","last_change":"2026-02-22T22:34:03.052873+0000","last_active":"2026-02-22T22:35:04.845344+0000","last_peered":"2026-02-22T22:35:04.845344+0000","last_clean":"2026-02-22T22:35:04.845344+0000","last_became_active":"2026-02-22T22:34:00.867119+0000","last_became_peered":"2026-02-22T22:34:00.867119+0000","last_unstale":"2026-02-22T22:35:04.845344+0000","last_undegraded":"2026-02-22T22:35:04.845344+0000","last_fullsized":"2026-02-22T22:35:04.845344+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:47:54.266607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025084499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.843978+0000","last_change":"2026-02-22T22:34:03.052868+0000","last_active":"2026-02-22T22:35:04.843978+0000","last_peered":"2026-02-22T22:35:04.843978+0000","last_clean":"2026-02-22T22:35:04.843978+0000","last_became_active":"2026-02-22T22:34:00.867120+0000","last_became_peered":"2026-02-22T22:34:00.867120+0000","last_unstale":"2026-02-22T22:35:04.843978+0000","last_undegraded":"2026-02-22T22:35:04.843978+0000","last_fullsized":"2026-02-22T22:35:04.843978+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:02:27.662737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028151199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":60,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.872632+0000","last_change":"2026-02-22T22:34:02.867985+0000","last_active":"2026-02-22T22:35:04.872632+0000","last_peered":"2026-02-22T22:35:04.872632+0000","last_clean":"2026-02-22T22:35:04.872632+0000","last_became_active":"2026-02-22T22:34:00.866689+0000","last_became_peered":"2026-02-22T22:34:00.866689+0000","last_unstale":"2026-02-22T22:35:04.872632+0000","last_undegraded":"2026-02-22T22:35:04.872632+0000","last_fullsized":"2026-02-22T22:35:04.872632+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:56:32.650734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050022800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":150,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.873784+0000","last_change":"2026-02-22T22:33:58.085668+0000","last_active":"2026-02-22T22:35:04.873784+0000","last_peered":"2026-02-22T22:35:04.873784+0000","last_clean":"2026-02-22T22:35:04.873784+0000","last_became_active":"2026-02-22T22:33:58.084957+0000","last_became_peered":"2026-02-22T22:33:58.084957+0000","last_unstale":"2026-02-22T22:35:04.873784+0000","last_undegraded":"2026-02-22T22:35:04.873784+0000","last_fullsized":"2026-02-22T22:35:04.873784+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_clean_scrub_stamp":"2026-02-22T22:33:56.854877+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:52:32.982688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705689,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":33428,"kb_used_data":2308,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94338412,"statfs":{"total":96636764160,"available":96602533888,"internally_reserved":0,"allocated":2363392,"data_stored":2150663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705688,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":34288,"kb_used_data":3168,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94337552,"statfs":{"total":96636764160,"available":96601653248,"internally_reserved":0,"allocated":3244032,"data_stored":3027638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738393,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":33700,"kb_used_data":2576,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94338140,"statfs":{"total":96636764160,"available":96602255360,"internally_reserved":0,"allocated":2637824,"data_stored":2411230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:35:45.759 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:35:45.918 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:35:45.918 INFO:teuthology.orchestra.run.trial198.stderr:dumped all 2026-02-22T22:35:45.926 INFO:teuthology.orchestra.run.trial198.stdout:{"pg_ready":true,"pg_map":{"version":93,"stamp":"2026-02-22T22:35:44.759947+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":101416,"kb_used_data":8052,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283014104,"statfs":{"total":289910292480,"available":289806442496,"internally_reserved":0,"allocated":8245248,"data_stored":7589531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-566223189,"num_objects":-237,"num_object_clones":0,"num_object_copies":-474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-237,"num_whiteouts":0,"num_read":-4099,"num_read_kb":-4299,"num_write":-974,"num_write_kb":-3525,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.062275"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874578+0000","last_change":"2026-02-22T22:34:02.868487+0000","last_active":"2026-02-22T22:35:04.874578+0000","last_peered":"2026-02-22T22:35:04.874578+0000","last_clean":"2026-02-22T22:35:04.874578+0000","last_became_active":"2026-02-22T22:34:00.867294+0000","last_became_peered":"2026-02-22T22:34:00.867294+0000","last_unstale":"2026-02-22T22:35:04.874578+0000","last_undegraded":"2026-02-22T22:35:04.874578+0000","last_fullsized":"2026-02-22T22:35:04.874578+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:49.554408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094867900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874665+0000","last_change":"2026-02-22T22:34:02.868266+0000","last_active":"2026-02-22T22:35:04.874665+0000","last_peered":"2026-02-22T22:35:04.874665+0000","last_clean":"2026-02-22T22:35:04.874665+0000","last_became_active":"2026-02-22T22:34:00.867282+0000","last_became_peered":"2026-02-22T22:34:00.867282+0000","last_unstale":"2026-02-22T22:35:04.874665+0000","last_undegraded":"2026-02-22T22:35:04.874665+0000","last_fullsized":"2026-02-22T22:35:04.874665+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:32:35.739961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072328099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874572+0000","last_change":"2026-02-22T22:34:02.868262+0000","last_active":"2026-02-22T22:35:04.874572+0000","last_peered":"2026-02-22T22:35:04.874572+0000","last_clean":"2026-02-22T22:35:04.874572+0000","last_became_active":"2026-02-22T22:34:00.867317+0000","last_became_peered":"2026-02-22T22:34:00.867317+0000","last_unstale":"2026-02-22T22:35:04.874572+0000","last_undegraded":"2026-02-22T22:35:04.874572+0000","last_fullsized":"2026-02-22T22:35:04.874572+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:39:15.990906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075292700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.873588+0000","last_change":"2026-02-22T22:34:02.867946+0000","last_active":"2026-02-22T22:35:04.873588+0000","last_peered":"2026-02-22T22:35:04.873588+0000","last_clean":"2026-02-22T22:35:04.873588+0000","last_became_active":"2026-02-22T22:34:00.867325+0000","last_became_peered":"2026-02-22T22:34:00.867325+0000","last_unstale":"2026-02-22T22:35:04.873588+0000","last_undegraded":"2026-02-22T22:35:04.873588+0000","last_fullsized":"2026-02-22T22:35:04.873588+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:29:12.873846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043021699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":62,"reported_epoch":38,"state":"active+clean","last_fresh":"2026-02-22T22:35:42.764066+0000","last_change":"2026-02-22T22:34:02.869362+0000","last_active":"2026-02-22T22:35:42.764066+0000","last_peered":"2026-02-22T22:35:42.764066+0000","last_clean":"2026-02-22T22:35:42.764066+0000","last_became_active":"2026-02-22T22:34:00.867085+0000","last_became_peered":"2026-02-22T22:34:00.867085+0000","last_unstale":"2026-02-22T22:35:42.764066+0000","last_undegraded":"2026-02-22T22:35:42.764066+0000","last_fullsized":"2026-02-22T22:35:42.764066+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:00:12.510734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089501999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.845344+0000","last_change":"2026-02-22T22:34:03.052873+0000","last_active":"2026-02-22T22:35:04.845344+0000","last_peered":"2026-02-22T22:35:04.845344+0000","last_clean":"2026-02-22T22:35:04.845344+0000","last_became_active":"2026-02-22T22:34:00.867119+0000","last_became_peered":"2026-02-22T22:34:00.867119+0000","last_unstale":"2026-02-22T22:35:04.845344+0000","last_undegraded":"2026-02-22T22:35:04.845344+0000","last_fullsized":"2026-02-22T22:35:04.845344+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:47:54.266607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025084499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.843978+0000","last_change":"2026-02-22T22:34:03.052868+0000","last_active":"2026-02-22T22:35:04.843978+0000","last_peered":"2026-02-22T22:35:04.843978+0000","last_clean":"2026-02-22T22:35:04.843978+0000","last_became_active":"2026-02-22T22:34:00.867120+0000","last_became_peered":"2026-02-22T22:34:00.867120+0000","last_unstale":"2026-02-22T22:35:04.843978+0000","last_undegraded":"2026-02-22T22:35:04.843978+0000","last_fullsized":"2026-02-22T22:35:04.843978+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:02:27.662737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028151199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":60,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.872632+0000","last_change":"2026-02-22T22:34:02.867985+0000","last_active":"2026-02-22T22:35:04.872632+0000","last_peered":"2026-02-22T22:35:04.872632+0000","last_clean":"2026-02-22T22:35:04.872632+0000","last_became_active":"2026-02-22T22:34:00.866689+0000","last_became_peered":"2026-02-22T22:34:00.866689+0000","last_unstale":"2026-02-22T22:35:04.872632+0000","last_undegraded":"2026-02-22T22:35:04.872632+0000","last_fullsized":"2026-02-22T22:35:04.872632+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:56:32.650734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050022800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":150,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.873784+0000","last_change":"2026-02-22T22:33:58.085668+0000","last_active":"2026-02-22T22:35:04.873784+0000","last_peered":"2026-02-22T22:35:04.873784+0000","last_clean":"2026-02-22T22:35:04.873784+0000","last_became_active":"2026-02-22T22:33:58.084957+0000","last_became_peered":"2026-02-22T22:33:58.084957+0000","last_unstale":"2026-02-22T22:35:04.873784+0000","last_undegraded":"2026-02-22T22:35:04.873784+0000","last_fullsized":"2026-02-22T22:35:04.873784+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_clean_scrub_stamp":"2026-02-22T22:33:56.854877+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:52:32.982688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705689,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":33428,"kb_used_data":2308,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94338412,"statfs":{"total":96636764160,"available":96602533888,"internally_reserved":0,"allocated":2363392,"data_stored":2150663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705688,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":34288,"kb_used_data":3168,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94337552,"statfs":{"total":96636764160,"available":96601653248,"internally_reserved":0,"allocated":3244032,"data_stored":3027638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738393,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":33700,"kb_used_data":2576,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94338140,"statfs":{"total":96636764160,"available":96602255360,"internally_reserved":0,"allocated":2637824,"data_stored":2411230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:35:45.928 INFO:tasks.ceph.ceph_manager.cluster2:clean! 2026-02-22T22:35:45.928 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:35:46.088 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:35:46.088 INFO:teuthology.orchestra.run.trial198.stderr:dumped all 2026-02-22T22:35:46.096 INFO:teuthology.orchestra.run.trial198.stdout:{"pg_ready":true,"pg_map":{"version":93,"stamp":"2026-02-22T22:35:44.759947+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":274,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":101416,"kb_used_data":8052,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283014104,"statfs":{"total":289910292480,"available":289806442496,"internally_reserved":0,"allocated":8245248,"data_stored":7589531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-566223189,"num_objects":-237,"num_object_clones":0,"num_object_copies":-474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-237,"num_whiteouts":0,"num_read":-4099,"num_read_kb":-4299,"num_write":-974,"num_write_kb":-3525,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.062275"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874578+0000","last_change":"2026-02-22T22:34:02.868487+0000","last_active":"2026-02-22T22:35:04.874578+0000","last_peered":"2026-02-22T22:35:04.874578+0000","last_clean":"2026-02-22T22:35:04.874578+0000","last_became_active":"2026-02-22T22:34:00.867294+0000","last_became_peered":"2026-02-22T22:34:00.867294+0000","last_unstale":"2026-02-22T22:35:04.874578+0000","last_undegraded":"2026-02-22T22:35:04.874578+0000","last_fullsized":"2026-02-22T22:35:04.874578+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:49.554408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094867900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874665+0000","last_change":"2026-02-22T22:34:02.868266+0000","last_active":"2026-02-22T22:35:04.874665+0000","last_peered":"2026-02-22T22:35:04.874665+0000","last_clean":"2026-02-22T22:35:04.874665+0000","last_became_active":"2026-02-22T22:34:00.867282+0000","last_became_peered":"2026-02-22T22:34:00.867282+0000","last_unstale":"2026-02-22T22:35:04.874665+0000","last_undegraded":"2026-02-22T22:35:04.874665+0000","last_fullsized":"2026-02-22T22:35:04.874665+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:32:35.739961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072328099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.874572+0000","last_change":"2026-02-22T22:34:02.868262+0000","last_active":"2026-02-22T22:35:04.874572+0000","last_peered":"2026-02-22T22:35:04.874572+0000","last_clean":"2026-02-22T22:35:04.874572+0000","last_became_active":"2026-02-22T22:34:00.867317+0000","last_became_peered":"2026-02-22T22:34:00.867317+0000","last_unstale":"2026-02-22T22:35:04.874572+0000","last_undegraded":"2026-02-22T22:35:04.874572+0000","last_fullsized":"2026-02-22T22:35:04.874572+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:39:15.990906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075292700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.873588+0000","last_change":"2026-02-22T22:34:02.867946+0000","last_active":"2026-02-22T22:35:04.873588+0000","last_peered":"2026-02-22T22:35:04.873588+0000","last_clean":"2026-02-22T22:35:04.873588+0000","last_became_active":"2026-02-22T22:34:00.867325+0000","last_became_peered":"2026-02-22T22:34:00.867325+0000","last_unstale":"2026-02-22T22:35:04.873588+0000","last_undegraded":"2026-02-22T22:35:04.873588+0000","last_fullsized":"2026-02-22T22:35:04.873588+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:29:12.873846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043021699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":62,"reported_epoch":38,"state":"active+clean","last_fresh":"2026-02-22T22:35:42.764066+0000","last_change":"2026-02-22T22:34:02.869362+0000","last_active":"2026-02-22T22:35:42.764066+0000","last_peered":"2026-02-22T22:35:42.764066+0000","last_clean":"2026-02-22T22:35:42.764066+0000","last_became_active":"2026-02-22T22:34:00.867085+0000","last_became_peered":"2026-02-22T22:34:00.867085+0000","last_unstale":"2026-02-22T22:35:42.764066+0000","last_undegraded":"2026-02-22T22:35:42.764066+0000","last_fullsized":"2026-02-22T22:35:42.764066+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:00:12.510734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089501999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.845344+0000","last_change":"2026-02-22T22:34:03.052873+0000","last_active":"2026-02-22T22:35:04.845344+0000","last_peered":"2026-02-22T22:35:04.845344+0000","last_clean":"2026-02-22T22:35:04.845344+0000","last_became_active":"2026-02-22T22:34:00.867119+0000","last_became_peered":"2026-02-22T22:34:00.867119+0000","last_unstale":"2026-02-22T22:35:04.845344+0000","last_undegraded":"2026-02-22T22:35:04.845344+0000","last_fullsized":"2026-02-22T22:35:04.845344+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:47:54.266607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025084499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":59,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.843978+0000","last_change":"2026-02-22T22:34:03.052868+0000","last_active":"2026-02-22T22:35:04.843978+0000","last_peered":"2026-02-22T22:35:04.843978+0000","last_clean":"2026-02-22T22:35:04.843978+0000","last_became_active":"2026-02-22T22:34:00.867120+0000","last_became_peered":"2026-02-22T22:34:00.867120+0000","last_unstale":"2026-02-22T22:35:04.843978+0000","last_undegraded":"2026-02-22T22:35:04.843978+0000","last_fullsized":"2026-02-22T22:35:04.843978+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:02:27.662737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028151199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":60,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.872632+0000","last_change":"2026-02-22T22:34:02.867985+0000","last_active":"2026-02-22T22:35:04.872632+0000","last_peered":"2026-02-22T22:35:04.872632+0000","last_clean":"2026-02-22T22:35:04.872632+0000","last_became_active":"2026-02-22T22:34:00.866689+0000","last_became_peered":"2026-02-22T22:34:00.866689+0000","last_unstale":"2026-02-22T22:35:04.872632+0000","last_undegraded":"2026-02-22T22:35:04.872632+0000","last_fullsized":"2026-02-22T22:35:04.872632+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:56:32.650734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050022800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":150,"reported_epoch":37,"state":"active+clean","last_fresh":"2026-02-22T22:35:04.873784+0000","last_change":"2026-02-22T22:33:58.085668+0000","last_active":"2026-02-22T22:35:04.873784+0000","last_peered":"2026-02-22T22:35:04.873784+0000","last_clean":"2026-02-22T22:35:04.873784+0000","last_became_active":"2026-02-22T22:33:58.084957+0000","last_became_peered":"2026-02-22T22:33:58.084957+0000","last_unstale":"2026-02-22T22:35:04.873784+0000","last_undegraded":"2026-02-22T22:35:04.873784+0000","last_fullsized":"2026-02-22T22:35:04.873784+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_clean_scrub_stamp":"2026-02-22T22:33:56.854877+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:52:32.982688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705689,"num_pgs":82,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":33428,"kb_used_data":2308,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94338412,"statfs":{"total":96636764160,"available":96602533888,"internally_reserved":0,"allocated":2363392,"data_stored":2150663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705688,"num_pgs":97,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":34288,"kb_used_data":3168,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94337552,"statfs":{"total":96636764160,"available":96601653248,"internally_reserved":0,"allocated":3244032,"data_stored":3027638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738393,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":33700,"kb_used_data":2576,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94338140,"statfs":{"total":96636764160,"available":96602255360,"internally_reserved":0,"allocated":2637824,"data_stored":2411230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:35:46.098 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd dump --format=json 2026-02-22T22:35:46.259 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:35:46.259 INFO:teuthology.orchestra.run.trial198.stdout:{"epoch":39,"fsid":"3163850b-09a4-4a3d-86d6-c3514600e52a","created":"2026-02-22T22:33:44.838704+0000","modified":"2026-02-22T22:35:44.758731+0000","last_up_change":"2026-02-22T22:33:54.853009+0000","last_in_change":"2026-02-22T22:33:45.414296+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":5,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:33:56.704026+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"13","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:33:59.029125+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"17","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":17,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"bb727418-5617-4c65-9bbe-c41f7c1c38f8","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6800","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6801","nonce":638091441}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6802","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6803","nonce":638091441}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6806","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6807","nonce":638091441}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6804","nonce":638091441},{"type":"v1","addr":"10.20.193.198:6805","nonce":638091441}]},"public_addr":"10.20.193.198:6801/638091441","cluster_addr":"10.20.193.198:6803/638091441","heartbeat_back_addr":"10.20.193.198:6807/638091441","heartbeat_front_addr":"10.20.193.198:6805/638091441","state":["exists","up"]},{"osd":1,"uuid":"8a10c63e-6aa0-48eb-b9b7-76b88bed0f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6808","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6809","nonce":3890729408}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6810","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6811","nonce":3890729408}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6814","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6815","nonce":3890729408}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6812","nonce":3890729408},{"type":"v1","addr":"10.20.193.198:6813","nonce":3890729408}]},"public_addr":"10.20.193.198:6809/3890729408","cluster_addr":"10.20.193.198:6811/3890729408","heartbeat_back_addr":"10.20.193.198:6815/3890729408","heartbeat_front_addr":"10.20.193.198:6813/3890729408","state":["exists","up"]},{"osd":2,"uuid":"95e8a862-6b24-4b97-bf8b-318f2d869686","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":9,"up_thru":27,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6816","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6817","nonce":2341346929}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6818","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6819","nonce":2341346929}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6822","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6823","nonce":2341346929}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.198:6820","nonce":2341346929},{"type":"v1","addr":"10.20.193.198:6821","nonce":2341346929}]},"public_addr":"10.20.193.198:6817/2341346929","cluster_addr":"10.20.193.198:6819/2341346929","heartbeat_back_addr":"10.20.193.198:6823/2341346929","heartbeat_front_addr":"10.20.193.198:6821/2341346929","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.789007+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.886670+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:52.844872+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:35:47.269 INFO:tasks.ceph:Scrubbing cluster2.osd.0 2026-02-22T22:35:47.270 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:35:47.351 INFO:teuthology.orchestra.run.trial198.stdout:{ 2026-02-22T22:35:47.352 INFO:teuthology.orchestra.run.trial198.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:35:47.352 INFO:teuthology.orchestra.run.trial198.stdout:} 2026-02-22T22:35:47.357 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 0 2026-02-22T22:35:47.517 INFO:teuthology.orchestra.run.trial198.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:35:47.525 INFO:tasks.ceph:Scrubbing cluster2.osd.1 2026-02-22T22:35:47.526 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:35:47.607 INFO:teuthology.orchestra.run.trial198.stdout:{ 2026-02-22T22:35:47.607 INFO:teuthology.orchestra.run.trial198.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:35:47.607 INFO:teuthology.orchestra.run.trial198.stdout:} 2026-02-22T22:35:47.612 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 1 2026-02-22T22:35:47.771 INFO:teuthology.orchestra.run.trial198.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:35:47.781 INFO:tasks.ceph:Scrubbing cluster2.osd.2 2026-02-22T22:35:47.781 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:35:47.861 INFO:teuthology.orchestra.run.trial198.stdout:{ 2026-02-22T22:35:47.861 INFO:teuthology.orchestra.run.trial198.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:35:47.861 INFO:teuthology.orchestra.run.trial198.stdout:} 2026-02-22T22:35:47.867 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 osd deep-scrub 2 2026-02-22T22:35:48.022 INFO:teuthology.orchestra.run.trial198.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:35:48.031 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:35:48.189 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:35:48.190 INFO:teuthology.orchestra.run.trial198.stderr:dumped all 2026-02-22T22:35:48.197 INFO:teuthology.orchestra.run.trial198.stdout:{"pg_ready":true,"pg_map":{"version":94,"stamp":"2026-02-22T22:35:46.699175+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":154,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":98980,"kb_used_data":5608,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283016540,"statfs":{"total":289910292480,"available":289808936960,"internally_reserved":0,"allocated":5742592,"data_stored":5096356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-566223189,"num_objects":-237,"num_object_clones":0,"num_object_copies":-474,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-237,"num_whiteouts":0,"num_read":-4099,"num_read_kb":-4299,"num_write":-974,"num_write_kb":-3525,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-4,"num_bytes_recovered":0,"num_keys_recovered":-5,"num_objects_omap":-29,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.001225"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":63,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:44.905523+0000","last_change":"2026-02-22T22:34:02.868487+0000","last_active":"2026-02-22T22:35:44.905523+0000","last_peered":"2026-02-22T22:35:44.905523+0000","last_clean":"2026-02-22T22:35:44.905523+0000","last_became_active":"2026-02-22T22:34:00.867294+0000","last_became_peered":"2026-02-22T22:34:00.867294+0000","last_unstale":"2026-02-22T22:35:44.905523+0000","last_undegraded":"2026-02-22T22:35:44.905523+0000","last_fullsized":"2026-02-22T22:35:44.905523+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:28:49.554408+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094867900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":63,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:44.907035+0000","last_change":"2026-02-22T22:34:02.868266+0000","last_active":"2026-02-22T22:35:44.907035+0000","last_peered":"2026-02-22T22:35:44.907035+0000","last_clean":"2026-02-22T22:35:44.907035+0000","last_became_active":"2026-02-22T22:34:00.867282+0000","last_became_peered":"2026-02-22T22:34:00.867282+0000","last_unstale":"2026-02-22T22:35:44.907035+0000","last_undegraded":"2026-02-22T22:35:44.907035+0000","last_fullsized":"2026-02-22T22:35:44.907035+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:32:35.739961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072328099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":63,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:44.906368+0000","last_change":"2026-02-22T22:34:02.868262+0000","last_active":"2026-02-22T22:35:44.906368+0000","last_peered":"2026-02-22T22:35:44.906368+0000","last_clean":"2026-02-22T22:35:44.906368+0000","last_became_active":"2026-02-22T22:34:00.867317+0000","last_became_peered":"2026-02-22T22:34:00.867317+0000","last_unstale":"2026-02-22T22:35:44.906368+0000","last_undegraded":"2026-02-22T22:35:44.906368+0000","last_fullsized":"2026-02-22T22:35:44.906368+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:39:15.990906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075292700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":63,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:44.904725+0000","last_change":"2026-02-22T22:34:02.867946+0000","last_active":"2026-02-22T22:35:44.904725+0000","last_peered":"2026-02-22T22:35:44.904725+0000","last_clean":"2026-02-22T22:35:44.904725+0000","last_became_active":"2026-02-22T22:34:00.867325+0000","last_became_peered":"2026-02-22T22:34:00.867325+0000","last_unstale":"2026-02-22T22:35:44.904725+0000","last_undegraded":"2026-02-22T22:35:44.904725+0000","last_fullsized":"2026-02-22T22:35:44.904725+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:29:12.873846+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043021699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"17'2","reported_seq":62,"reported_epoch":38,"state":"active+clean","last_fresh":"2026-02-22T22:35:42.764066+0000","last_change":"2026-02-22T22:34:02.869362+0000","last_active":"2026-02-22T22:35:42.764066+0000","last_peered":"2026-02-22T22:35:42.764066+0000","last_clean":"2026-02-22T22:35:42.764066+0000","last_became_active":"2026-02-22T22:34:00.867085+0000","last_became_peered":"2026-02-22T22:34:00.867085+0000","last_unstale":"2026-02-22T22:35:42.764066+0000","last_undegraded":"2026-02-22T22:35:42.764066+0000","last_fullsized":"2026-02-22T22:35:42.764066+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:00:12.510734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089501999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":63,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:44.765690+0000","last_change":"2026-02-22T22:34:03.052873+0000","last_active":"2026-02-22T22:35:44.765690+0000","last_peered":"2026-02-22T22:35:44.765690+0000","last_clean":"2026-02-22T22:35:44.765690+0000","last_became_active":"2026-02-22T22:34:00.867119+0000","last_became_peered":"2026-02-22T22:34:00.867119+0000","last_unstale":"2026-02-22T22:35:44.765690+0000","last_undegraded":"2026-02-22T22:35:44.765690+0000","last_fullsized":"2026-02-22T22:35:44.765690+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:47:54.266607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025084499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":63,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:44.763359+0000","last_change":"2026-02-22T22:34:03.052868+0000","last_active":"2026-02-22T22:35:44.763359+0000","last_peered":"2026-02-22T22:35:44.763359+0000","last_clean":"2026-02-22T22:35:44.763359+0000","last_became_active":"2026-02-22T22:34:00.867120+0000","last_became_peered":"2026-02-22T22:34:00.867120+0000","last_unstale":"2026-02-22T22:35:44.763359+0000","last_undegraded":"2026-02-22T22:35:44.763359+0000","last_fullsized":"2026-02-22T22:35:44.763359+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:02:27.662737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028151199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"15'1","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:44.903477+0000","last_change":"2026-02-22T22:34:02.867985+0000","last_active":"2026-02-22T22:35:44.903477+0000","last_peered":"2026-02-22T22:35:44.903477+0000","last_clean":"2026-02-22T22:35:44.903477+0000","last_became_active":"2026-02-22T22:34:00.866689+0000","last_became_peered":"2026-02-22T22:34:00.866689+0000","last_unstale":"2026-02-22T22:35:44.903477+0000","last_undegraded":"2026-02-22T22:35:44.903477+0000","last_fullsized":"2026-02-22T22:35:44.903477+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:59.859851+0000","last_clean_scrub_stamp":"2026-02-22T22:33:59.859851+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:56:32.650734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050022800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"12'72","reported_seq":154,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:44.906165+0000","last_change":"2026-02-22T22:33:58.085668+0000","last_active":"2026-02-22T22:35:44.906165+0000","last_peered":"2026-02-22T22:35:44.906165+0000","last_clean":"2026-02-22T22:35:44.906165+0000","last_became_active":"2026-02-22T22:33:58.084957+0000","last_became_peered":"2026-02-22T22:33:58.084957+0000","last_unstale":"2026-02-22T22:35:44.906165+0000","last_undegraded":"2026-02-22T22:35:44.906165+0000","last_fullsized":"2026-02-22T22:35:44.906165+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:56.854877+0000","last_clean_scrub_stamp":"2026-02-22T22:33:56.854877+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:52:32.982688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705690,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31452,"kb_used_data":328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94340388,"statfs":{"total":96636764160,"available":96604557312,"internally_reserved":0,"allocated":335872,"data_stored":136730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705689,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":33828,"kb_used_data":2704,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94338012,"statfs":{"total":96636764160,"available":96602124288,"internally_reserved":0,"allocated":2768896,"data_stored":2548396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738393,"num_pgs":95,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":33700,"kb_used_data":2576,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94338140,"statfs":{"total":96636764160,"available":96602255360,"internally_reserved":0,"allocated":2637824,"data_stored":2411230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:35:48.202 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T22:33:59.859851+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.202 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T22:33:59.859851+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.203 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T22:33:59.859851+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.203 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T22:33:59.859851+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.203 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T22:33:59.859851+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.203 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T22:33:59.859851+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.203 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T22:33:59.859851+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.203 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T22:33:59.859851+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=59, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.203 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T22:33:56.854877+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=56, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=35, tm_sec=46, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:35:48.204 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:36:08.205 DEBUG:teuthology.orchestra.run.trial198:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster2 pg dump --format=json 2026-02-22T22:36:08.365 INFO:teuthology.orchestra.run.trial198.stdout: 2026-02-22T22:36:08.366 INFO:teuthology.orchestra.run.trial198.stderr:dumped all 2026-02-22T22:36:08.373 INFO:teuthology.orchestra.run.trial198.stdout:{"pg_ready":true,"pg_map":{"version":104,"stamp":"2026-02-22T22:36:06.700705+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95512,"kb_used_data":2152,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283020008,"statfs":{"total":289910292480,"available":289812488192,"internally_reserved":0,"allocated":2203648,"data_stored":1590964,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000878"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":72,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:52.870460+0000","last_change":"2026-02-22T22:35:52.870473+0000","last_active":"2026-02-22T22:35:52.870460+0000","last_peered":"2026-02-22T22:35:52.870460+0000","last_clean":"2026-02-22T22:35:52.870460+0000","last_became_active":"2026-02-22T22:34:00.867294+0000","last_became_peered":"2026-02-22T22:34:00.867294+0000","last_unstale":"2026-02-22T22:35:52.870460+0000","last_undegraded":"2026-02-22T22:35:52.870460+0000","last_fullsized":"2026-02-22T22:35:52.870460+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:35:52.870415+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:35:52.870415+0000","last_clean_scrub_stamp":"2026-02-22T22:35:52.870415+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:22:07.830155+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00094867900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":72,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:50.890735+0000","last_change":"2026-02-22T22:35:50.890743+0000","last_active":"2026-02-22T22:35:50.890735+0000","last_peered":"2026-02-22T22:35:50.890735+0000","last_clean":"2026-02-22T22:35:50.890735+0000","last_became_active":"2026-02-22T22:34:00.867282+0000","last_became_peered":"2026-02-22T22:34:00.867282+0000","last_unstale":"2026-02-22T22:35:50.890735+0000","last_undegraded":"2026-02-22T22:35:50.890735+0000","last_fullsized":"2026-02-22T22:35:50.890735+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:35:50.890702+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:35:50.890702+0000","last_clean_scrub_stamp":"2026-02-22T22:35:50.890702+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:04:02.830556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00072328099999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":73,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:49.883696+0000","last_change":"2026-02-22T22:35:49.883696+0000","last_active":"2026-02-22T22:35:49.883696+0000","last_peered":"2026-02-22T22:35:49.883696+0000","last_clean":"2026-02-22T22:35:49.883696+0000","last_became_active":"2026-02-22T22:34:00.867317+0000","last_became_peered":"2026-02-22T22:34:00.867317+0000","last_unstale":"2026-02-22T22:35:49.883696+0000","last_undegraded":"2026-02-22T22:35:49.883696+0000","last_fullsized":"2026-02-22T22:35:49.883696+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:35:49.883641+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:35:49.883641+0000","last_clean_scrub_stamp":"2026-02-22T22:35:49.883641+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:03:27.535628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00075292700000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":73,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:51.873373+0000","last_change":"2026-02-22T22:35:51.873373+0000","last_active":"2026-02-22T22:35:51.873373+0000","last_peered":"2026-02-22T22:35:51.873373+0000","last_clean":"2026-02-22T22:35:51.873373+0000","last_became_active":"2026-02-22T22:34:00.867325+0000","last_became_peered":"2026-02-22T22:34:00.867325+0000","last_unstale":"2026-02-22T22:35:51.873373+0000","last_undegraded":"2026-02-22T22:35:51.873373+0000","last_fullsized":"2026-02-22T22:35:51.873373+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:35:51.873326+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:35:51.873326+0000","last_clean_scrub_stamp":"2026-02-22T22:35:51.873326+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:46:20.630388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00043021699999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"39'3","reported_seq":74,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:48.284721+0000","last_change":"2026-02-22T22:35:48.284735+0000","last_active":"2026-02-22T22:35:48.284721+0000","last_peered":"2026-02-22T22:35:48.284721+0000","last_clean":"2026-02-22T22:35:48.284721+0000","last_became_active":"2026-02-22T22:34:00.867085+0000","last_became_peered":"2026-02-22T22:34:00.867085+0000","last_unstale":"2026-02-22T22:35:48.284721+0000","last_undegraded":"2026-02-22T22:35:48.284721+0000","last_fullsized":"2026-02-22T22:35:48.284721+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"39'3","last_scrub_stamp":"2026-02-22T22:35:48.284680+0000","last_deep_scrub":"39'3","last_deep_scrub_stamp":"2026-02-22T22:35:48.284680+0000","last_clean_scrub_stamp":"2026-02-22T22:35:48.284680+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:30:13.224015+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00089501999999999995,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":73,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:48.915749+0000","last_change":"2026-02-22T22:35:48.915749+0000","last_active":"2026-02-22T22:35:48.915749+0000","last_peered":"2026-02-22T22:35:48.915749+0000","last_clean":"2026-02-22T22:35:48.915749+0000","last_became_active":"2026-02-22T22:34:00.867119+0000","last_became_peered":"2026-02-22T22:34:00.867119+0000","last_unstale":"2026-02-22T22:35:48.915749+0000","last_undegraded":"2026-02-22T22:35:48.915749+0000","last_fullsized":"2026-02-22T22:35:48.915749+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:35:48.915656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:35:48.915656+0000","last_clean_scrub_stamp":"2026-02-22T22:35:48.915656+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T07:42:23.077806+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00025084499999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":73,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:49.912819+0000","last_change":"2026-02-22T22:35:49.912819+0000","last_active":"2026-02-22T22:35:49.912819+0000","last_peered":"2026-02-22T22:35:49.912819+0000","last_clean":"2026-02-22T22:35:49.912819+0000","last_became_active":"2026-02-22T22:34:00.867120+0000","last_became_peered":"2026-02-22T22:34:00.867120+0000","last_unstale":"2026-02-22T22:35:49.912819+0000","last_undegraded":"2026-02-22T22:35:49.912819+0000","last_fullsized":"2026-02-22T22:35:49.912819+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:35:49.912773+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:35:49.912773+0000","last_clean_scrub_stamp":"2026-02-22T22:35:49.912773+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T09:19:19.611336+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00028151199999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"39'2","reported_seq":75,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:48.837119+0000","last_change":"2026-02-22T22:35:48.837119+0000","last_active":"2026-02-22T22:35:48.837119+0000","last_peered":"2026-02-22T22:35:48.837119+0000","last_clean":"2026-02-22T22:35:48.837119+0000","last_became_active":"2026-02-22T22:34:00.866689+0000","last_became_peered":"2026-02-22T22:34:00.866689+0000","last_unstale":"2026-02-22T22:35:48.837119+0000","last_undegraded":"2026-02-22T22:35:48.837119+0000","last_fullsized":"2026-02-22T22:35:48.837119+0000","mapping_epoch":14,"log_start":"0'0","ondisk_log_start":"0'0","created":14,"last_epoch_clean":15,"parent":"0.0","parent_split_bits":0,"last_scrub":"39'2","last_scrub_stamp":"2026-02-22T22:35:48.837063+0000","last_deep_scrub":"39'2","last_deep_scrub_stamp":"2026-02-22T22:35:48.837063+0000","last_clean_scrub_stamp":"2026-02-22T22:35:48.837063+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:24:57.418248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00050022800000000002,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"39'74","reported_seq":166,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:47.867801+0000","last_change":"2026-02-22T22:35:47.867801+0000","last_active":"2026-02-22T22:35:47.867801+0000","last_peered":"2026-02-22T22:35:47.867801+0000","last_clean":"2026-02-22T22:35:47.867801+0000","last_became_active":"2026-02-22T22:33:58.084957+0000","last_became_peered":"2026-02-22T22:33:58.084957+0000","last_unstale":"2026-02-22T22:35:47.867801+0000","last_undegraded":"2026-02-22T22:35:47.867801+0000","last_fullsized":"2026-02-22T22:35:47.867801+0000","mapping_epoch":11,"log_start":"0'0","ondisk_log_start":"0'0","created":11,"last_epoch_clean":12,"parent":"0.0","parent_split_bits":0,"last_scrub":"39'74","last_scrub_stamp":"2026-02-22T22:35:47.867750+0000","last_deep_scrub":"39'74","last_deep_scrub_stamp":"2026-02-22T22:35:47.867750+0000","last_clean_scrub_stamp":"2026-02-22T22:35:47.867750+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:16:29.782989+0000","scrub_duration":2,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":9,"seq":38654705694,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31448,"kb_used_data":328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94340392,"statfs":{"total":96636764160,"available":96604561408,"internally_reserved":0,"allocated":335872,"data_stored":136730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":9,"seq":38654705693,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32032,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339808,"statfs":{"total":96636764160,"available":96603963392,"internally_reserved":0,"allocated":933888,"data_stored":727117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738397,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32032,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339808,"statfs":{"total":96636764160,"available":96603963392,"internally_reserved":0,"allocated":933888,"data_stored":727117,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:36:08.375 DEBUG:teuthology.orchestra.run.trial198:> sudo ceph --cluster cluster2 config set global mon_health_to_clog false 2026-02-22T22:36:08.545 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:36:08.546 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:36:08.546 DEBUG:tasks.ceph.cluster2.osd.0:waiting for process to exit 2026-02-22T22:36:08.546 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:08.585 INFO:tasks.ceph.cluster2.osd.0:Stopped 2026-02-22T22:36:08.585 DEBUG:tasks.ceph.cluster2.osd.1:waiting for process to exit 2026-02-22T22:36:08.586 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:08.660 INFO:tasks.ceph.cluster2.osd.1:Stopped 2026-02-22T22:36:08.660 DEBUG:tasks.ceph.cluster2.osd.2:waiting for process to exit 2026-02-22T22:36:08.661 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:08.727 INFO:tasks.ceph.cluster2.osd.2:Stopped 2026-02-22T22:36:08.728 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:36:08.728 DEBUG:tasks.ceph.cluster2.mgr.x:waiting for process to exit 2026-02-22T22:36:08.728 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:08.769 INFO:tasks.ceph.cluster2.mgr.x:Stopped 2026-02-22T22:36:08.769 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:36:08.769 DEBUG:tasks.ceph.cluster2.mon.a:waiting for process to exit 2026-02-22T22:36:08.770 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:08.779 INFO:tasks.ceph.cluster2.mon.a:Stopped 2026-02-22T22:36:08.779 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:36:08.779 DEBUG:teuthology.orchestra.run.trial198:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster2.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:36:08.846 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-0 on ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:36:08.846 DEBUG:teuthology.orchestra.run.trial198:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-0 2026-02-22T22:36:08.914 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-1 on ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:36:08.914 DEBUG:teuthology.orchestra.run.trial198:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-1 2026-02-22T22:36:08.982 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster2-2 on ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:36:08.983 DEBUG:teuthology.orchestra.run.trial198:> sync && sudo umount -f /var/lib/ceph/osd/cluster2-2 2026-02-22T22:36:09.099 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:36:09.100 DEBUG:teuthology.misc:Transferring archived files from trial198:/var/lib/ceph/mon/cluster2-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242/data/cluster2.mon.a.tgz 2026-02-22T22:36:09.101 DEBUG:teuthology.orchestra.run.trial198:> mktemp 2026-02-22T22:36:09.118 INFO:teuthology.orchestra.run.trial198.stdout:/tmp/tmp.dWQUGSKg5H 2026-02-22T22:36:09.118 DEBUG:teuthology.orchestra.run.trial198:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster2-a -- . > /tmp/tmp.dWQUGSKg5H 2026-02-22T22:36:09.208 DEBUG:teuthology.orchestra.run.trial198:> sudo chmod 0666 /tmp/tmp.dWQUGSKg5H 2026-02-22T22:36:09.288 DEBUG:teuthology.orchestra.remote:trial198:/tmp/tmp.dWQUGSKg5H is 206KB 2026-02-22T22:36:09.377 DEBUG:teuthology.orchestra.run.trial198:> rm -fr /tmp/tmp.dWQUGSKg5H 2026-02-22T22:36:09.394 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:36:09.395 DEBUG:teuthology.orchestra.run.trial194:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:36:09.397 DEBUG:teuthology.orchestra.run.trial198:> sudo rm -rf -- /etc/ceph/cluster2.conf /etc/ceph/cluster2.keyring /home/ubuntu/cephtest/cluster2.data /home/ubuntu/cephtest/cluster2.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:36:09.457 DEBUG:teuthology.run_tasks:Unwinding manager ceph 2026-02-22T22:36:09.474 INFO:tasks.ceph.ceph_manager.cluster1:waiting for clean 2026-02-22T22:36:09.474 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:36:09.634 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:36:09.635 INFO:teuthology.orchestra.run.trial194.stderr:dumped all 2026-02-22T22:36:09.643 INFO:teuthology.orchestra.run.trial194.stdout:{"pg_ready":true,"pg_map":{"version":125,"stamp":"2026-02-22T22:36:08.003082+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95524,"kb_used_data":2152,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283019996,"statfs":{"total":289910292480,"available":289812475904,"internally_reserved":0,"allocated":2203648,"data_stored":1589479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000845"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856728+0000","last_change":"2026-02-22T22:33:21.783670+0000","last_active":"2026-02-22T22:35:43.856728+0000","last_peered":"2026-02-22T22:35:43.856728+0000","last_clean":"2026-02-22T22:35:43.856728+0000","last_became_active":"2026-02-22T22:33:19.779975+0000","last_became_peered":"2026-02-22T22:33:19.779975+0000","last_unstale":"2026-02-22T22:35:43.856728+0000","last_undegraded":"2026-02-22T22:35:43.856728+0000","last_fullsized":"2026-02-22T22:35:43.856728+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:41:03.479104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085832499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856287+0000","last_change":"2026-02-22T22:33:21.783880+0000","last_active":"2026-02-22T22:35:43.856287+0000","last_peered":"2026-02-22T22:35:43.856287+0000","last_clean":"2026-02-22T22:35:43.856287+0000","last_became_active":"2026-02-22T22:33:19.779948+0000","last_became_peered":"2026-02-22T22:33:19.779948+0000","last_unstale":"2026-02-22T22:35:43.856287+0000","last_undegraded":"2026-02-22T22:35:43.856287+0000","last_fullsized":"2026-02-22T22:35:43.856287+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:43.595634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001084284,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.857197+0000","last_change":"2026-02-22T22:33:21.783616+0000","last_active":"2026-02-22T22:35:43.857197+0000","last_peered":"2026-02-22T22:35:43.857197+0000","last_clean":"2026-02-22T22:35:43.857197+0000","last_became_active":"2026-02-22T22:33:19.779809+0000","last_became_peered":"2026-02-22T22:33:19.779809+0000","last_unstale":"2026-02-22T22:35:43.857197+0000","last_undegraded":"2026-02-22T22:35:43.857197+0000","last_fullsized":"2026-02-22T22:35:43.857197+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:07.737454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078483800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.855627+0000","last_change":"2026-02-22T22:33:21.783394+0000","last_active":"2026-02-22T22:35:43.855627+0000","last_peered":"2026-02-22T22:35:43.855627+0000","last_clean":"2026-02-22T22:35:43.855627+0000","last_became_active":"2026-02-22T22:33:19.779840+0000","last_became_peered":"2026-02-22T22:33:19.779840+0000","last_unstale":"2026-02-22T22:35:43.855627+0000","last_undegraded":"2026-02-22T22:35:43.855627+0000","last_fullsized":"2026-02-22T22:35:43.855627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:39:30.893086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000614622,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":67,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:45.626460+0000","last_change":"2026-02-22T22:33:21.781441+0000","last_active":"2026-02-22T22:35:45.626460+0000","last_peered":"2026-02-22T22:35:45.626460+0000","last_clean":"2026-02-22T22:35:45.626460+0000","last_became_active":"2026-02-22T22:33:19.780216+0000","last_became_peered":"2026-02-22T22:33:19.780216+0000","last_unstale":"2026-02-22T22:35:45.626460+0000","last_undegraded":"2026-02-22T22:35:45.626460+0000","last_fullsized":"2026-02-22T22:35:45.626460+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:18:55.136618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049561099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:46.618833+0000","last_change":"2026-02-22T22:33:21.780155+0000","last_active":"2026-02-22T22:35:46.618833+0000","last_peered":"2026-02-22T22:35:46.618833+0000","last_clean":"2026-02-22T22:35:46.618833+0000","last_became_active":"2026-02-22T22:33:19.780151+0000","last_became_peered":"2026-02-22T22:33:19.780151+0000","last_unstale":"2026-02-22T22:35:46.618833+0000","last_undegraded":"2026-02-22T22:35:46.618833+0000","last_fullsized":"2026-02-22T22:35:46.618833+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:58:34.778892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029422600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:46.618173+0000","last_change":"2026-02-22T22:33:21.780152+0000","last_active":"2026-02-22T22:35:46.618173+0000","last_peered":"2026-02-22T22:35:46.618173+0000","last_clean":"2026-02-22T22:35:46.618173+0000","last_became_active":"2026-02-22T22:33:19.780160+0000","last_became_peered":"2026-02-22T22:33:19.780160+0000","last_unstale":"2026-02-22T22:35:46.618173+0000","last_undegraded":"2026-02-22T22:35:46.618173+0000","last_fullsized":"2026-02-22T22:35:46.618173+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:25:02.825325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034237599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":67,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856900+0000","last_change":"2026-02-22T22:33:21.783221+0000","last_active":"2026-02-22T22:35:43.856900+0000","last_peered":"2026-02-22T22:35:43.856900+0000","last_clean":"2026-02-22T22:35:43.856900+0000","last_became_active":"2026-02-22T22:33:19.779118+0000","last_became_peered":"2026-02-22T22:33:19.779118+0000","last_unstale":"2026-02-22T22:35:43.856900+0000","last_undegraded":"2026-02-22T22:35:43.856900+0000","last_fullsized":"2026-02-22T22:35:43.856900+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:07:18.496977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049514999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":157,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.855605+0000","last_change":"2026-02-22T22:33:15.817593+0000","last_active":"2026-02-22T22:35:43.855605+0000","last_peered":"2026-02-22T22:35:43.855605+0000","last_clean":"2026-02-22T22:35:43.855605+0000","last_became_active":"2026-02-22T22:33:15.817125+0000","last_became_peered":"2026-02-22T22:33:15.817125+0000","last_unstale":"2026-02-22T22:35:43.855605+0000","last_undegraded":"2026-02-22T22:35:43.855605+0000","last_fullsized":"2026-02-22T22:35:43.855605+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_clean_scrub_stamp":"2026-02-22T22:33:14.765836+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:24:54.631757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738404,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31452,"kb_used_data":328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94340388,"statfs":{"total":96636764160,"available":96604557312,"internally_reserved":0,"allocated":335872,"data_stored":136235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738405,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32036,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339804,"statfs":{"total":96636764160,"available":96603959296,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738404,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32036,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339804,"statfs":{"total":96636764160,"available":96603959296,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:36:09.645 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:36:09.807 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:36:09.808 INFO:teuthology.orchestra.run.trial194.stderr:dumped all 2026-02-22T22:36:09.815 INFO:teuthology.orchestra.run.trial194.stdout:{"pg_ready":true,"pg_map":{"version":125,"stamp":"2026-02-22T22:36:08.003082+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95524,"kb_used_data":2152,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283019996,"statfs":{"total":289910292480,"available":289812475904,"internally_reserved":0,"allocated":2203648,"data_stored":1589479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000845"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856728+0000","last_change":"2026-02-22T22:33:21.783670+0000","last_active":"2026-02-22T22:35:43.856728+0000","last_peered":"2026-02-22T22:35:43.856728+0000","last_clean":"2026-02-22T22:35:43.856728+0000","last_became_active":"2026-02-22T22:33:19.779975+0000","last_became_peered":"2026-02-22T22:33:19.779975+0000","last_unstale":"2026-02-22T22:35:43.856728+0000","last_undegraded":"2026-02-22T22:35:43.856728+0000","last_fullsized":"2026-02-22T22:35:43.856728+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:41:03.479104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085832499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856287+0000","last_change":"2026-02-22T22:33:21.783880+0000","last_active":"2026-02-22T22:35:43.856287+0000","last_peered":"2026-02-22T22:35:43.856287+0000","last_clean":"2026-02-22T22:35:43.856287+0000","last_became_active":"2026-02-22T22:33:19.779948+0000","last_became_peered":"2026-02-22T22:33:19.779948+0000","last_unstale":"2026-02-22T22:35:43.856287+0000","last_undegraded":"2026-02-22T22:35:43.856287+0000","last_fullsized":"2026-02-22T22:35:43.856287+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:43.595634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001084284,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.857197+0000","last_change":"2026-02-22T22:33:21.783616+0000","last_active":"2026-02-22T22:35:43.857197+0000","last_peered":"2026-02-22T22:35:43.857197+0000","last_clean":"2026-02-22T22:35:43.857197+0000","last_became_active":"2026-02-22T22:33:19.779809+0000","last_became_peered":"2026-02-22T22:33:19.779809+0000","last_unstale":"2026-02-22T22:35:43.857197+0000","last_undegraded":"2026-02-22T22:35:43.857197+0000","last_fullsized":"2026-02-22T22:35:43.857197+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:07.737454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078483800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.855627+0000","last_change":"2026-02-22T22:33:21.783394+0000","last_active":"2026-02-22T22:35:43.855627+0000","last_peered":"2026-02-22T22:35:43.855627+0000","last_clean":"2026-02-22T22:35:43.855627+0000","last_became_active":"2026-02-22T22:33:19.779840+0000","last_became_peered":"2026-02-22T22:33:19.779840+0000","last_unstale":"2026-02-22T22:35:43.855627+0000","last_undegraded":"2026-02-22T22:35:43.855627+0000","last_fullsized":"2026-02-22T22:35:43.855627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:39:30.893086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000614622,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":67,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:45.626460+0000","last_change":"2026-02-22T22:33:21.781441+0000","last_active":"2026-02-22T22:35:45.626460+0000","last_peered":"2026-02-22T22:35:45.626460+0000","last_clean":"2026-02-22T22:35:45.626460+0000","last_became_active":"2026-02-22T22:33:19.780216+0000","last_became_peered":"2026-02-22T22:33:19.780216+0000","last_unstale":"2026-02-22T22:35:45.626460+0000","last_undegraded":"2026-02-22T22:35:45.626460+0000","last_fullsized":"2026-02-22T22:35:45.626460+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:18:55.136618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049561099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:46.618833+0000","last_change":"2026-02-22T22:33:21.780155+0000","last_active":"2026-02-22T22:35:46.618833+0000","last_peered":"2026-02-22T22:35:46.618833+0000","last_clean":"2026-02-22T22:35:46.618833+0000","last_became_active":"2026-02-22T22:33:19.780151+0000","last_became_peered":"2026-02-22T22:33:19.780151+0000","last_unstale":"2026-02-22T22:35:46.618833+0000","last_undegraded":"2026-02-22T22:35:46.618833+0000","last_fullsized":"2026-02-22T22:35:46.618833+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:58:34.778892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029422600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:46.618173+0000","last_change":"2026-02-22T22:33:21.780152+0000","last_active":"2026-02-22T22:35:46.618173+0000","last_peered":"2026-02-22T22:35:46.618173+0000","last_clean":"2026-02-22T22:35:46.618173+0000","last_became_active":"2026-02-22T22:33:19.780160+0000","last_became_peered":"2026-02-22T22:33:19.780160+0000","last_unstale":"2026-02-22T22:35:46.618173+0000","last_undegraded":"2026-02-22T22:35:46.618173+0000","last_fullsized":"2026-02-22T22:35:46.618173+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:25:02.825325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034237599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":67,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856900+0000","last_change":"2026-02-22T22:33:21.783221+0000","last_active":"2026-02-22T22:35:43.856900+0000","last_peered":"2026-02-22T22:35:43.856900+0000","last_clean":"2026-02-22T22:35:43.856900+0000","last_became_active":"2026-02-22T22:33:19.779118+0000","last_became_peered":"2026-02-22T22:33:19.779118+0000","last_unstale":"2026-02-22T22:35:43.856900+0000","last_undegraded":"2026-02-22T22:35:43.856900+0000","last_fullsized":"2026-02-22T22:35:43.856900+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:07:18.496977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049514999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":157,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.855605+0000","last_change":"2026-02-22T22:33:15.817593+0000","last_active":"2026-02-22T22:35:43.855605+0000","last_peered":"2026-02-22T22:35:43.855605+0000","last_clean":"2026-02-22T22:35:43.855605+0000","last_became_active":"2026-02-22T22:33:15.817125+0000","last_became_peered":"2026-02-22T22:33:15.817125+0000","last_unstale":"2026-02-22T22:35:43.855605+0000","last_undegraded":"2026-02-22T22:35:43.855605+0000","last_fullsized":"2026-02-22T22:35:43.855605+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_clean_scrub_stamp":"2026-02-22T22:33:14.765836+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:24:54.631757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738404,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31452,"kb_used_data":328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94340388,"statfs":{"total":96636764160,"available":96604557312,"internally_reserved":0,"allocated":335872,"data_stored":136235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738405,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32036,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339804,"statfs":{"total":96636764160,"available":96603959296,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738404,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32036,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339804,"statfs":{"total":96636764160,"available":96603959296,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:36:09.817 INFO:tasks.ceph.ceph_manager.cluster1:clean! 2026-02-22T22:36:09.817 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:36:09.976 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:36:09.976 INFO:teuthology.orchestra.run.trial194.stderr:dumped all 2026-02-22T22:36:09.984 INFO:teuthology.orchestra.run.trial194.stdout:{"pg_ready":true,"pg_map":{"version":125,"stamp":"2026-02-22T22:36:08.003082+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95524,"kb_used_data":2152,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283019996,"statfs":{"total":289910292480,"available":289812475904,"internally_reserved":0,"allocated":2203648,"data_stored":1589479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000845"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856728+0000","last_change":"2026-02-22T22:33:21.783670+0000","last_active":"2026-02-22T22:35:43.856728+0000","last_peered":"2026-02-22T22:35:43.856728+0000","last_clean":"2026-02-22T22:35:43.856728+0000","last_became_active":"2026-02-22T22:33:19.779975+0000","last_became_peered":"2026-02-22T22:33:19.779975+0000","last_unstale":"2026-02-22T22:35:43.856728+0000","last_undegraded":"2026-02-22T22:35:43.856728+0000","last_fullsized":"2026-02-22T22:35:43.856728+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:41:03.479104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085832499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856287+0000","last_change":"2026-02-22T22:33:21.783880+0000","last_active":"2026-02-22T22:35:43.856287+0000","last_peered":"2026-02-22T22:35:43.856287+0000","last_clean":"2026-02-22T22:35:43.856287+0000","last_became_active":"2026-02-22T22:33:19.779948+0000","last_became_peered":"2026-02-22T22:33:19.779948+0000","last_unstale":"2026-02-22T22:35:43.856287+0000","last_undegraded":"2026-02-22T22:35:43.856287+0000","last_fullsized":"2026-02-22T22:35:43.856287+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:43.595634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001084284,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.857197+0000","last_change":"2026-02-22T22:33:21.783616+0000","last_active":"2026-02-22T22:35:43.857197+0000","last_peered":"2026-02-22T22:35:43.857197+0000","last_clean":"2026-02-22T22:35:43.857197+0000","last_became_active":"2026-02-22T22:33:19.779809+0000","last_became_peered":"2026-02-22T22:33:19.779809+0000","last_unstale":"2026-02-22T22:35:43.857197+0000","last_undegraded":"2026-02-22T22:35:43.857197+0000","last_fullsized":"2026-02-22T22:35:43.857197+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:07.737454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078483800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.855627+0000","last_change":"2026-02-22T22:33:21.783394+0000","last_active":"2026-02-22T22:35:43.855627+0000","last_peered":"2026-02-22T22:35:43.855627+0000","last_clean":"2026-02-22T22:35:43.855627+0000","last_became_active":"2026-02-22T22:33:19.779840+0000","last_became_peered":"2026-02-22T22:33:19.779840+0000","last_unstale":"2026-02-22T22:35:43.855627+0000","last_undegraded":"2026-02-22T22:35:43.855627+0000","last_fullsized":"2026-02-22T22:35:43.855627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:39:30.893086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000614622,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":67,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:45.626460+0000","last_change":"2026-02-22T22:33:21.781441+0000","last_active":"2026-02-22T22:35:45.626460+0000","last_peered":"2026-02-22T22:35:45.626460+0000","last_clean":"2026-02-22T22:35:45.626460+0000","last_became_active":"2026-02-22T22:33:19.780216+0000","last_became_peered":"2026-02-22T22:33:19.780216+0000","last_unstale":"2026-02-22T22:35:45.626460+0000","last_undegraded":"2026-02-22T22:35:45.626460+0000","last_fullsized":"2026-02-22T22:35:45.626460+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:18:55.136618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049561099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:46.618833+0000","last_change":"2026-02-22T22:33:21.780155+0000","last_active":"2026-02-22T22:35:46.618833+0000","last_peered":"2026-02-22T22:35:46.618833+0000","last_clean":"2026-02-22T22:35:46.618833+0000","last_became_active":"2026-02-22T22:33:19.780151+0000","last_became_peered":"2026-02-22T22:33:19.780151+0000","last_unstale":"2026-02-22T22:35:46.618833+0000","last_undegraded":"2026-02-22T22:35:46.618833+0000","last_fullsized":"2026-02-22T22:35:46.618833+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:58:34.778892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029422600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:46.618173+0000","last_change":"2026-02-22T22:33:21.780152+0000","last_active":"2026-02-22T22:35:46.618173+0000","last_peered":"2026-02-22T22:35:46.618173+0000","last_clean":"2026-02-22T22:35:46.618173+0000","last_became_active":"2026-02-22T22:33:19.780160+0000","last_became_peered":"2026-02-22T22:33:19.780160+0000","last_unstale":"2026-02-22T22:35:46.618173+0000","last_undegraded":"2026-02-22T22:35:46.618173+0000","last_fullsized":"2026-02-22T22:35:46.618173+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:25:02.825325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034237599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":67,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856900+0000","last_change":"2026-02-22T22:33:21.783221+0000","last_active":"2026-02-22T22:35:43.856900+0000","last_peered":"2026-02-22T22:35:43.856900+0000","last_clean":"2026-02-22T22:35:43.856900+0000","last_became_active":"2026-02-22T22:33:19.779118+0000","last_became_peered":"2026-02-22T22:33:19.779118+0000","last_unstale":"2026-02-22T22:35:43.856900+0000","last_undegraded":"2026-02-22T22:35:43.856900+0000","last_fullsized":"2026-02-22T22:35:43.856900+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:07:18.496977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049514999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":157,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.855605+0000","last_change":"2026-02-22T22:33:15.817593+0000","last_active":"2026-02-22T22:35:43.855605+0000","last_peered":"2026-02-22T22:35:43.855605+0000","last_clean":"2026-02-22T22:35:43.855605+0000","last_became_active":"2026-02-22T22:33:15.817125+0000","last_became_peered":"2026-02-22T22:33:15.817125+0000","last_unstale":"2026-02-22T22:35:43.855605+0000","last_undegraded":"2026-02-22T22:35:43.855605+0000","last_fullsized":"2026-02-22T22:35:43.855605+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_clean_scrub_stamp":"2026-02-22T22:33:14.765836+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:24:54.631757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738404,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31452,"kb_used_data":328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94340388,"statfs":{"total":96636764160,"available":96604557312,"internally_reserved":0,"allocated":335872,"data_stored":136235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738405,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32036,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339804,"statfs":{"total":96636764160,"available":96603959296,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738404,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32036,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339804,"statfs":{"total":96636764160,"available":96603959296,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:36:09.986 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd dump --format=json 2026-02-22T22:36:10.144 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:36:10.144 INFO:teuthology.orchestra.run.trial194.stdout:{"epoch":39,"fsid":"53304493-37aa-4143-9c19-51b919f55963","created":"2026-02-22T22:33:03.668145+0000","modified":"2026-02-22T22:35:43.849929+0000","last_up_change":"2026-02-22T22:33:12.761432+0000","last_in_change":"2026-02-22T22:33:04.254047+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":4,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":3,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"squid","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2026-02-22T22:33:13.995981+0000","flags":1,"flags_names":"hashpspool","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"12","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2.9900000095367432,"score_stable":2.9900000095367432,"optimal_score":0.67000001668930054,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2026-02-22T22:33:17.872309+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":2,"min_size":1,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"is_stretch_pool":false,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"16","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":2,"snap_epoch":16,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":1.8799999952316284,"score_stable":1.8799999952316284,"optimal_score":1,"raw_score_acting":1.8799999952316284,"raw_score_stable":1.8799999952316284,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"952bf1aa-0b8e-4788-b6ee-4090a0547472","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6802","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6804","nonce":286964534}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6806","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6808","nonce":286964534}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6814","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6816","nonce":286964534}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6810","nonce":286964534},{"type":"v1","addr":"10.20.193.194:6812","nonce":286964534}]},"public_addr":"10.20.193.194:6804/286964534","cluster_addr":"10.20.193.194:6808/286964534","heartbeat_back_addr":"10.20.193.194:6816/286964534","heartbeat_front_addr":"10.20.193.194:6812/286964534","state":["exists","up"]},{"osd":1,"uuid":"8bc82af6-3bff-4b7b-96e2-53757ee66e72","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6803","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6805","nonce":3688961049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6807","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6809","nonce":3688961049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6815","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6817","nonce":3688961049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6811","nonce":3688961049},{"type":"v1","addr":"10.20.193.194:6813","nonce":3688961049}]},"public_addr":"10.20.193.194:6805/3688961049","cluster_addr":"10.20.193.194:6809/3688961049","heartbeat_back_addr":"10.20.193.194:6817/3688961049","heartbeat_front_addr":"10.20.193.194:6813/3688961049","state":["exists","up"]},{"osd":2,"uuid":"ec376c34-9c21-490d-a010-6ee8ac90b04d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":35,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6818","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6819","nonce":1852489993}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6820","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6821","nonce":1852489993}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6824","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6825","nonce":1852489993}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"10.20.193.194:6822","nonce":1852489993},{"type":"v1","addr":"10.20.193.194:6823","nonce":1852489993}]},"public_addr":"10.20.193.194:6819/1852489993","cluster_addr":"10.20.193.194:6821/1852489993","heartbeat_back_addr":"10.20.193.194:6825/1852489993","heartbeat_front_addr":"10.20.193.194:6823/1852489993","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.706930+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.729631+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2026-02-22T22:33:11.737583+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2026-02-22T22:36:11.153 INFO:tasks.ceph:Scrubbing cluster1.osd.0 2026-02-22T22:36:11.153 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.0 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:36:11.236 INFO:teuthology.orchestra.run.trial194.stdout:{ 2026-02-22T22:36:11.236 INFO:teuthology.orchestra.run.trial194.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:36:11.237 INFO:teuthology.orchestra.run.trial194.stdout:} 2026-02-22T22:36:11.244 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 0 2026-02-22T22:36:11.404 INFO:teuthology.orchestra.run.trial194.stderr:instructed osd(s) 0 to deep-scrub 2026-02-22T22:36:11.413 INFO:tasks.ceph:Scrubbing cluster1.osd.1 2026-02-22T22:36:11.413 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.1 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:36:11.493 INFO:teuthology.orchestra.run.trial194.stdout:{ 2026-02-22T22:36:11.494 INFO:teuthology.orchestra.run.trial194.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:36:11.494 INFO:teuthology.orchestra.run.trial194.stdout:} 2026-02-22T22:36:11.500 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 1 2026-02-22T22:36:11.657 INFO:teuthology.orchestra.run.trial194.stderr:instructed osd(s) 1 to deep-scrub 2026-02-22T22:36:11.665 INFO:tasks.ceph:Scrubbing cluster1.osd.2 2026-02-22T22:36:11.666 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 tell osd.2 config set osd_debug_deep_scrub_sleep 0 2026-02-22T22:36:11.746 INFO:teuthology.orchestra.run.trial194.stdout:{ 2026-02-22T22:36:11.746 INFO:teuthology.orchestra.run.trial194.stdout: "success": "osd_debug_deep_scrub_sleep = '' (not observed, change may require restart) osd_delete_sleep = '' osd_delete_sleep_hdd = '' osd_delete_sleep_hybrid = '' osd_delete_sleep_ssd = '' osd_max_backfills = '' osd_recovery_max_active = '' osd_recovery_max_active_hdd = '' osd_recovery_max_active_ssd = '' osd_recovery_sleep = '' osd_recovery_sleep_degraded = '' osd_recovery_sleep_degraded_hdd = '' osd_recovery_sleep_degraded_hybrid = '' osd_recovery_sleep_degraded_ssd = '' osd_recovery_sleep_hdd = '' osd_recovery_sleep_hybrid = '' osd_recovery_sleep_ssd = '' osd_scrub_sleep = '' osd_snap_trim_sleep = '' osd_snap_trim_sleep_hdd = '' osd_snap_trim_sleep_hybrid = '' osd_snap_trim_sleep_ssd = '' " 2026-02-22T22:36:11.746 INFO:teuthology.orchestra.run.trial194.stdout:} 2026-02-22T22:36:11.752 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 osd deep-scrub 2 2026-02-22T22:36:11.910 INFO:teuthology.orchestra.run.trial194.stderr:instructed osd(s) 2 to deep-scrub 2026-02-22T22:36:11.919 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:36:12.078 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:36:12.078 INFO:teuthology.orchestra.run.trial194.stderr:dumped all 2026-02-22T22:36:12.086 INFO:teuthology.orchestra.run.trial194.stdout:{"pg_ready":true,"pg_map":{"version":127,"stamp":"2026-02-22T22:36:12.003427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":75,"ondisk_log_size":75,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95524,"kb_used_data":2152,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283019996,"statfs":{"total":289910292480,"available":289812475904,"internally_reserved":0,"allocated":2203648,"data_stored":1589479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000897"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856728+0000","last_change":"2026-02-22T22:33:21.783670+0000","last_active":"2026-02-22T22:35:43.856728+0000","last_peered":"2026-02-22T22:35:43.856728+0000","last_clean":"2026-02-22T22:35:43.856728+0000","last_became_active":"2026-02-22T22:33:19.779975+0000","last_became_peered":"2026-02-22T22:33:19.779975+0000","last_unstale":"2026-02-22T22:35:43.856728+0000","last_undegraded":"2026-02-22T22:35:43.856728+0000","last_fullsized":"2026-02-22T22:35:43.856728+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T06:41:03.479104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085832499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856287+0000","last_change":"2026-02-22T22:33:21.783880+0000","last_active":"2026-02-22T22:35:43.856287+0000","last_peered":"2026-02-22T22:35:43.856287+0000","last_clean":"2026-02-22T22:35:43.856287+0000","last_became_active":"2026-02-22T22:33:19.779948+0000","last_became_peered":"2026-02-22T22:33:19.779948+0000","last_unstale":"2026-02-22T22:35:43.856287+0000","last_undegraded":"2026-02-22T22:35:43.856287+0000","last_fullsized":"2026-02-22T22:35:43.856287+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T22:36:43.595634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001084284,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.857197+0000","last_change":"2026-02-22T22:33:21.783616+0000","last_active":"2026-02-22T22:35:43.857197+0000","last_peered":"2026-02-22T22:35:43.857197+0000","last_clean":"2026-02-22T22:35:43.857197+0000","last_became_active":"2026-02-22T22:33:19.779809+0000","last_became_peered":"2026-02-22T22:33:19.779809+0000","last_unstale":"2026-02-22T22:35:43.857197+0000","last_undegraded":"2026-02-22T22:35:43.857197+0000","last_fullsized":"2026-02-22T22:35:43.857197+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:32:07.737454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078483800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":66,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.855627+0000","last_change":"2026-02-22T22:33:21.783394+0000","last_active":"2026-02-22T22:35:43.855627+0000","last_peered":"2026-02-22T22:35:43.855627+0000","last_clean":"2026-02-22T22:35:43.855627+0000","last_became_active":"2026-02-22T22:33:19.779840+0000","last_became_peered":"2026-02-22T22:33:19.779840+0000","last_unstale":"2026-02-22T22:35:43.855627+0000","last_undegraded":"2026-02-22T22:35:43.855627+0000","last_fullsized":"2026-02-22T22:35:43.855627+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:39:30.893086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000614622,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"16'2","reported_seq":67,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:45.626460+0000","last_change":"2026-02-22T22:33:21.781441+0000","last_active":"2026-02-22T22:35:45.626460+0000","last_peered":"2026-02-22T22:35:45.626460+0000","last_clean":"2026-02-22T22:35:45.626460+0000","last_became_active":"2026-02-22T22:33:19.780216+0000","last_became_peered":"2026-02-22T22:33:19.780216+0000","last_unstale":"2026-02-22T22:35:45.626460+0000","last_undegraded":"2026-02-22T22:35:45.626460+0000","last_fullsized":"2026-02-22T22:35:45.626460+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:18:55.136618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049561099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:46.618833+0000","last_change":"2026-02-22T22:33:21.780155+0000","last_active":"2026-02-22T22:35:46.618833+0000","last_peered":"2026-02-22T22:35:46.618833+0000","last_clean":"2026-02-22T22:35:46.618833+0000","last_became_active":"2026-02-22T22:33:19.780151+0000","last_became_peered":"2026-02-22T22:33:19.780151+0000","last_unstale":"2026-02-22T22:35:46.618833+0000","last_undegraded":"2026-02-22T22:35:46.618833+0000","last_fullsized":"2026-02-22T22:35:46.618833+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:58:34.778892+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029422600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":64,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:46.618173+0000","last_change":"2026-02-22T22:33:21.780152+0000","last_active":"2026-02-22T22:35:46.618173+0000","last_peered":"2026-02-22T22:35:46.618173+0000","last_clean":"2026-02-22T22:35:46.618173+0000","last_became_active":"2026-02-22T22:33:19.780160+0000","last_became_peered":"2026-02-22T22:33:19.780160+0000","last_unstale":"2026-02-22T22:35:46.618173+0000","last_undegraded":"2026-02-22T22:35:46.618173+0000","last_fullsized":"2026-02-22T22:35:46.618173+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T10:25:02.825325+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034237599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"14'1","reported_seq":67,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.856900+0000","last_change":"2026-02-22T22:33:21.783221+0000","last_active":"2026-02-22T22:35:43.856900+0000","last_peered":"2026-02-22T22:35:43.856900+0000","last_clean":"2026-02-22T22:35:43.856900+0000","last_became_active":"2026-02-22T22:33:19.779118+0000","last_became_peered":"2026-02-22T22:33:19.779118+0000","last_unstale":"2026-02-22T22:35:43.856900+0000","last_undegraded":"2026-02-22T22:35:43.856900+0000","last_fullsized":"2026-02-22T22:35:43.856900+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:18.772243+0000","last_clean_scrub_stamp":"2026-02-22T22:33:18.772243+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:07:18.496977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049514999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"11'72","reported_seq":157,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:35:43.855605+0000","last_change":"2026-02-22T22:33:15.817593+0000","last_active":"2026-02-22T22:35:43.855605+0000","last_peered":"2026-02-22T22:35:43.855605+0000","last_clean":"2026-02-22T22:35:43.855605+0000","last_became_active":"2026-02-22T22:33:15.817125+0000","last_became_peered":"2026-02-22T22:33:15.817125+0000","last_unstale":"2026-02-22T22:35:43.855605+0000","last_undegraded":"2026-02-22T22:35:43.855605+0000","last_fullsized":"2026-02-22T22:35:43.855605+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:33:14.765836+0000","last_clean_scrub_stamp":"2026-02-22T22:33:14.765836+0000","objects_scrubbed":0,"log_size":72,"log_dups_size":0,"ondisk_log_size":72,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:24:54.631757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3,"ondisk_log_size":3,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":72,"ondisk_log_size":72,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738405,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31452,"kb_used_data":328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94340388,"statfs":{"total":96636764160,"available":96604557312,"internally_reserved":0,"allocated":335872,"data_stored":136235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738406,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32036,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339804,"statfs":{"total":96636764160,"available":96603959296,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738405,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32036,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339804,"statfs":{"total":96636764160,"available":96603959296,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:36:12.088 INFO:tasks.ceph:pgid 2.7 last_scrub_stamp 2026-02-22T22:33:18.772243+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=18, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.088 INFO:tasks.ceph:pgid 2.6 last_scrub_stamp 2026-02-22T22:33:18.772243+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=18, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.089 INFO:tasks.ceph:pgid 2.5 last_scrub_stamp 2026-02-22T22:33:18.772243+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=18, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.089 INFO:tasks.ceph:pgid 2.4 last_scrub_stamp 2026-02-22T22:33:18.772243+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=18, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.089 INFO:tasks.ceph:pgid 2.2 last_scrub_stamp 2026-02-22T22:33:18.772243+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=18, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.089 INFO:tasks.ceph:pgid 2.1 last_scrub_stamp 2026-02-22T22:33:18.772243+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=18, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.089 INFO:tasks.ceph:pgid 2.0 last_scrub_stamp 2026-02-22T22:33:18.772243+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=18, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.089 INFO:tasks.ceph:pgid 2.3 last_scrub_stamp 2026-02-22T22:33:18.772243+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=18, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.089 INFO:tasks.ceph:pgid 1.0 last_scrub_stamp 2026-02-22T22:33:14.765836+0000 time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=33, tm_sec=14, tm_wday=6, tm_yday=53, tm_isdst=-1) <= time.struct_time(tm_year=2026, tm_mon=2, tm_mday=22, tm_hour=22, tm_min=36, tm_sec=10, tm_wday=6, tm_yday=53, tm_isdst=0) 2026-02-22T22:36:12.090 INFO:tasks.ceph:Still waiting for all pgs to be scrubbed. 2026-02-22T22:36:32.090 DEBUG:teuthology.orchestra.run.trial194:> sudo adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 120 ceph --cluster cluster1 pg dump --format=json 2026-02-22T22:36:32.250 INFO:teuthology.orchestra.run.trial194.stdout: 2026-02-22T22:36:32.251 INFO:teuthology.orchestra.run.trial194.stderr:dumped all 2026-02-22T22:36:32.259 INFO:teuthology.orchestra.run.trial194.stdout:{"pg_ready":true,"pg_map":{"version":137,"stamp":"2026-02-22T22:36:32.004867+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590387,"num_objects":4,"num_object_clones":0,"num_object_copies":8,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":103,"num_write_kb":1630,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79,"ondisk_log_size":79,"up":18,"acting":18,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":18,"num_osds":3,"num_per_pool_osds":3,"num_per_pool_omap_osds":3,"kb":283115520,"kb_used":95512,"kb_used_data":2152,"kb_used_omap":4,"kb_used_meta":93307,"kb_avail":283020008,"statfs":{"total":289910292480,"available":289812488192,"internally_reserved":0,"allocated":2203648,"data_stored":1589479,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4782,"internal_metadata":95546706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.000742"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":75,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:16.788613+0000","last_change":"2026-02-22T22:36:16.788625+0000","last_active":"2026-02-22T22:36:16.788613+0000","last_peered":"2026-02-22T22:36:16.788613+0000","last_clean":"2026-02-22T22:36:16.788613+0000","last_became_active":"2026-02-22T22:33:19.779975+0000","last_became_peered":"2026-02-22T22:33:19.779975+0000","last_unstale":"2026-02-22T22:36:16.788613+0000","last_undegraded":"2026-02-22T22:36:16.788613+0000","last_fullsized":"2026-02-22T22:36:16.788613+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:36:16.788564+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:36:16.788564+0000","last_clean_scrub_stamp":"2026-02-22T22:36:16.788564+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T00:02:12.877316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00085832499999999995,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":75,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:15.759330+0000","last_change":"2026-02-22T22:36:15.759343+0000","last_active":"2026-02-22T22:36:15.759330+0000","last_peered":"2026-02-22T22:36:15.759330+0000","last_clean":"2026-02-22T22:36:15.759330+0000","last_became_active":"2026-02-22T22:33:19.779948+0000","last_became_peered":"2026-02-22T22:33:19.779948+0000","last_unstale":"2026-02-22T22:36:15.759330+0000","last_undegraded":"2026-02-22T22:36:15.759330+0000","last_fullsized":"2026-02-22T22:36:15.759330+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:36:15.759269+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:36:15.759269+0000","last_clean_scrub_stamp":"2026-02-22T22:36:15.759269+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T02:37:27.462031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001084284,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":75,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:12.802936+0000","last_change":"2026-02-22T22:36:12.802956+0000","last_active":"2026-02-22T22:36:12.802936+0000","last_peered":"2026-02-22T22:36:12.802936+0000","last_clean":"2026-02-22T22:36:12.802936+0000","last_became_active":"2026-02-22T22:33:19.779809+0000","last_became_peered":"2026-02-22T22:33:19.779809+0000","last_unstale":"2026-02-22T22:36:12.802936+0000","last_undegraded":"2026-02-22T22:36:12.802936+0000","last_fullsized":"2026-02-22T22:36:12.802936+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:36:12.802859+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:36:12.802859+0000","last_clean_scrub_stamp":"2026-02-22T22:36:12.802859+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:07:24.257263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078483800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":75,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:13.779965+0000","last_change":"2026-02-22T22:36:13.779979+0000","last_active":"2026-02-22T22:36:13.779965+0000","last_peered":"2026-02-22T22:36:13.779965+0000","last_clean":"2026-02-22T22:36:13.779965+0000","last_became_active":"2026-02-22T22:33:19.779840+0000","last_became_peered":"2026-02-22T22:33:19.779840+0000","last_unstale":"2026-02-22T22:36:13.779965+0000","last_undegraded":"2026-02-22T22:36:13.779965+0000","last_fullsized":"2026-02-22T22:36:13.779965+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:36:13.779898+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:36:13.779898+0000","last_clean_scrub_stamp":"2026-02-22T22:36:13.779898+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T01:48:43.232749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000614622,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.2","version":"39'3","reported_seq":77,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:12.202781+0000","last_change":"2026-02-22T22:36:12.202793+0000","last_active":"2026-02-22T22:36:12.202781+0000","last_peered":"2026-02-22T22:36:12.202781+0000","last_clean":"2026-02-22T22:36:12.202781+0000","last_became_active":"2026-02-22T22:33:19.780216+0000","last_became_peered":"2026-02-22T22:33:19.780216+0000","last_unstale":"2026-02-22T22:36:12.202781+0000","last_undegraded":"2026-02-22T22:36:12.202781+0000","last_fullsized":"2026-02-22T22:36:12.202781+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"39'3","last_scrub_stamp":"2026-02-22T22:36:12.202750+0000","last_deep_scrub":"39'3","last_deep_scrub_stamp":"2026-02-22T22:36:12.202750+0000","last_clean_scrub_stamp":"2026-02-22T22:36:12.202750+0000","objects_scrubbed":1,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-23T23:47:27.887949+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0.00049561099999999997,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":73,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:12.055308+0000","last_change":"2026-02-22T22:36:12.055319+0000","last_active":"2026-02-22T22:36:12.055308+0000","last_peered":"2026-02-22T22:36:12.055308+0000","last_clean":"2026-02-22T22:36:12.055308+0000","last_became_active":"2026-02-22T22:33:19.780151+0000","last_became_peered":"2026-02-22T22:33:19.780151+0000","last_unstale":"2026-02-22T22:36:12.055308+0000","last_undegraded":"2026-02-22T22:36:12.055308+0000","last_fullsized":"2026-02-22T22:36:12.055308+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:36:12.055257+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:36:12.055257+0000","last_clean_scrub_stamp":"2026-02-22T22:36:12.055257+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T03:53:42.322908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00029422600000000003,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":73,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:13.091578+0000","last_change":"2026-02-22T22:36:13.091598+0000","last_active":"2026-02-22T22:36:13.091578+0000","last_peered":"2026-02-22T22:36:13.091578+0000","last_clean":"2026-02-22T22:36:13.091578+0000","last_became_active":"2026-02-22T22:33:19.780160+0000","last_became_peered":"2026-02-22T22:33:19.780160+0000","last_unstale":"2026-02-22T22:36:13.091578+0000","last_undegraded":"2026-02-22T22:36:13.091578+0000","last_fullsized":"2026-02-22T22:36:13.091578+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2026-02-22T22:36:13.091501+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2026-02-22T22:36:13.091501+0000","last_clean_scrub_stamp":"2026-02-22T22:36:13.091501+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:53:07.654157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034237599999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.3","version":"39'2","reported_seq":77,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:14.788291+0000","last_change":"2026-02-22T22:36:14.788303+0000","last_active":"2026-02-22T22:36:14.788291+0000","last_peered":"2026-02-22T22:36:14.788291+0000","last_clean":"2026-02-22T22:36:14.788291+0000","last_became_active":"2026-02-22T22:33:19.779118+0000","last_became_peered":"2026-02-22T22:33:19.779118+0000","last_unstale":"2026-02-22T22:36:14.788291+0000","last_undegraded":"2026-02-22T22:36:14.788291+0000","last_fullsized":"2026-02-22T22:36:14.788291+0000","mapping_epoch":13,"log_start":"0'0","ondisk_log_start":"0'0","created":13,"last_epoch_clean":14,"parent":"0.0","parent_split_bits":0,"last_scrub":"39'2","last_scrub_stamp":"2026-02-22T22:36:14.788267+0000","last_deep_scrub":"39'2","last_deep_scrub_stamp":"2026-02-22T22:36:14.788267+0000","last_clean_scrub_stamp":"2026-02-22T22:36:14.788267+0000","objects_scrubbed":1,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T04:52:36.266317+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00049514999999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":2,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.0","version":"39'74","reported_seq":168,"reported_epoch":39,"state":"active+clean","last_fresh":"2026-02-22T22:36:11.823703+0000","last_change":"2026-02-22T22:36:11.823719+0000","last_active":"2026-02-22T22:36:11.823703+0000","last_peered":"2026-02-22T22:36:11.823703+0000","last_clean":"2026-02-22T22:36:11.823703+0000","last_became_active":"2026-02-22T22:33:15.817125+0000","last_became_peered":"2026-02-22T22:33:15.817125+0000","last_unstale":"2026-02-22T22:36:11.823703+0000","last_undegraded":"2026-02-22T22:36:11.823703+0000","last_fullsized":"2026-02-22T22:36:11.823703+0000","mapping_epoch":10,"log_start":"0'0","ondisk_log_start":"0'0","created":10,"last_epoch_clean":11,"parent":"0.0","parent_split_bits":0,"last_scrub":"39'74","last_scrub_stamp":"2026-02-22T22:36:11.823658+0000","last_deep_scrub":"39'74","last_deep_scrub_stamp":"2026-02-22T22:36:11.823658+0000","last_clean_scrub_stamp":"2026-02-22T22:36:11.823658+0000","objects_scrubbed":2,"log_size":74,"log_dups_size":0,"ondisk_log_size":74,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2026-02-24T08:57:46.371318+0000","scrub_duration":1,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":19,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2,"num_write_kb":2,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":38,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":5,"ondisk_log_size":5,"up":16,"acting":16,"num_store_stats":3},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":4,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":66,"num_read_kb":55,"num_write":101,"num_write_kb":1628,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1187840,"data_stored":1180736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74,"ondisk_log_size":74,"up":2,"acting":2,"num_store_stats":2}],"osd_stats":[{"osd":2,"up_from":8,"seq":34359738409,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":31448,"kb_used_data":328,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94340392,"statfs":{"total":96636764160,"available":96604561408,"internally_reserved":0,"allocated":335872,"data_stored":136235,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":8,"seq":34359738410,"num_pgs":9,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32032,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339808,"statfs":{"total":96636764160,"available":96603963392,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":8,"seq":34359738409,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":94371840,"kb_used":32032,"kb_used_data":912,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":94339808,"statfs":{"total":96636764160,"available":96603963392,"internally_reserved":0,"allocated":933888,"data_stored":726622,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2026-02-22T22:36:32.260 DEBUG:teuthology.orchestra.run.trial194:> sudo ceph --cluster cluster1 config set global mon_health_to_clog false 2026-02-22T22:36:32.425 INFO:teuthology.misc:Shutting down mds daemons... 2026-02-22T22:36:32.425 INFO:teuthology.misc:Shutting down osd daemons... 2026-02-22T22:36:32.425 DEBUG:tasks.ceph.cluster1.osd.0:waiting for process to exit 2026-02-22T22:36:32.425 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:32.455 INFO:tasks.ceph.cluster1.osd.0:Stopped 2026-02-22T22:36:32.455 DEBUG:tasks.ceph.cluster1.osd.1:waiting for process to exit 2026-02-22T22:36:32.455 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:32.532 INFO:tasks.ceph.cluster1.osd.1:Stopped 2026-02-22T22:36:32.532 DEBUG:tasks.ceph.cluster1.osd.2:waiting for process to exit 2026-02-22T22:36:32.532 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:32.598 INFO:tasks.ceph.cluster1.osd.2:Stopped 2026-02-22T22:36:32.598 INFO:teuthology.misc:Shutting down mgr daemons... 2026-02-22T22:36:32.599 DEBUG:tasks.ceph.cluster1.mgr.x:waiting for process to exit 2026-02-22T22:36:32.599 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:32.669 INFO:tasks.ceph.cluster1.mgr.x:Stopped 2026-02-22T22:36:32.669 INFO:teuthology.misc:Shutting down mon daemons... 2026-02-22T22:36:32.670 DEBUG:tasks.ceph.cluster1.mon.a:waiting for process to exit 2026-02-22T22:36:32.670 INFO:teuthology.orchestra.run:waiting for 300 2026-02-22T22:36:32.720 INFO:tasks.ceph.cluster1.mon.a:Stopped 2026-02-22T22:36:32.720 INFO:tasks.ceph:Checking cluster log for badness... 2026-02-22T22:36:32.721 DEBUG:teuthology.orchestra.run.trial194:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/cluster1.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | egrep -v '\(OSD_SLOW_PING_TIME' | head -n 1 2026-02-22T22:36:32.788 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-0 on ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:36:32.788 DEBUG:teuthology.orchestra.run.trial194:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-0 2026-02-22T22:36:32.907 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-1 on ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:36:32.907 DEBUG:teuthology.orchestra.run.trial194:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-1 2026-02-22T22:36:32.934 INFO:tasks.ceph:Unmounting /var/lib/ceph/osd/cluster1-2 on ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:36:32.934 DEBUG:teuthology.orchestra.run.trial194:> sync && sudo umount -f /var/lib/ceph/osd/cluster1-2 2026-02-22T22:36:33.002 INFO:tasks.ceph:Archiving mon data... 2026-02-22T22:36:33.003 DEBUG:teuthology.misc:Transferring archived files from trial194:/var/lib/ceph/mon/cluster1-a to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242/data/cluster1.mon.a.tgz 2026-02-22T22:36:33.003 DEBUG:teuthology.orchestra.run.trial194:> mktemp 2026-02-22T22:36:33.060 INFO:teuthology.orchestra.run.trial194.stdout:/tmp/tmp.C7QCvCuhf2 2026-02-22T22:36:33.061 DEBUG:teuthology.orchestra.run.trial194:> sudo tar cz -f - -C /var/lib/ceph/mon/cluster1-a -- . > /tmp/tmp.C7QCvCuhf2 2026-02-22T22:36:33.151 DEBUG:teuthology.orchestra.run.trial194:> sudo chmod 0666 /tmp/tmp.C7QCvCuhf2 2026-02-22T22:36:33.232 DEBUG:teuthology.orchestra.remote:trial194:/tmp/tmp.C7QCvCuhf2 is 214KB 2026-02-22T22:36:33.324 DEBUG:teuthology.orchestra.run.trial194:> rm -fr /tmp/tmp.C7QCvCuhf2 2026-02-22T22:36:33.339 INFO:tasks.ceph:Cleaning ceph cluster... 2026-02-22T22:36:33.340 DEBUG:teuthology.orchestra.run.trial194:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:36:33.382 DEBUG:teuthology.orchestra.run.trial198:> sudo rm -rf -- /etc/ceph/cluster1.conf /etc/ceph/cluster1.keyring /home/ubuntu/cephtest/cluster1.data /home/ubuntu/cephtest/cluster1.monmap /home/ubuntu/cephtest/../*.pid 2026-02-22T22:36:33.460 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:36:33.476 INFO:teuthology.util.scanner:summary_data or yaml_file is empty! 2026-02-22T22:36:33.477 INFO:tasks.ceph:Archiving crash dumps... 2026-02-22T22:36:33.479 DEBUG:teuthology.misc:Transferring archived files from trial194:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242/remote/trial194/crash 2026-02-22T22:36:33.480 DEBUG:teuthology.orchestra.run.trial194:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:36:33.525 DEBUG:teuthology.misc:Transferring archived files from trial198:/var/lib/ceph/crash to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242/remote/trial198/crash 2026-02-22T22:36:33.527 DEBUG:teuthology.orchestra.run.trial198:> sudo tar c -f - -C /var/lib/ceph/crash -- . 2026-02-22T22:36:33.550 INFO:tasks.ceph:Compressing logs... 2026-02-22T22:36:33.551 DEBUG:teuthology.orchestra.run.trial194:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:36:33.570 DEBUG:teuthology.orchestra.run.trial198:> time sudo find /var/log/ceph -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:36:33.592 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46173.log 2026-02-22T22:36:33.593 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46472.log 2026-02-22T22:36:33.593 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46173.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46173.log.gz 2026-02-22T22:36:33.593 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46738.log 2026-02-22T22:36:33.593 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46472.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47550.log 2026-02-22T22:36:33.593 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46472.log.gz 2026-02-22T22:36:33.594 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46738.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46325.log 2026-02-22T22:36:33.594 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46738.log.gz 2026-02-22T22:36:33.594 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5/var/log/ceph/cluster1-client.admin.47550.log: --verbose -- /var/log/ceph/cluster1-client.admin.42977.log 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47550.log.gz 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46325.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46325.log.gz 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mgr.x.log 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.42977.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mon.a.log 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42977.log.gz 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster2-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43219.log 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-mon.a.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.2.log 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.43219.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43219.log.gz 2026-02-22T22:36:33.606 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47844.log 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-osd.2.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45873.log 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47844.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47844.log.gz 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46859.log 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.45873.log: gzip -5 --verbose -- /var/log/ceph/cluster1-osd.0.log 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45873.log.gz 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46859.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46859.log.gz 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47697.log 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47648.log 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47697.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47697.log.gz 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.37843.log 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47648.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48038.log 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47648.log.gz 2026-02-22T22:36:33.607 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/ceph.tmp-client.admin.37843.log: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.37843.log.gz 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-osd.1.log 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.48038.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43151.log 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48038.log.gz 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47501.log 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.43151.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43151.log.gz 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46323.log 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47501.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.45964.log 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47501.log.gz 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46323.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46323.log.gz 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42930.log 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.45964.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46222.log 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.45964.log.gz 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.42930.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42930.log.gz 2026-02-22T22:36:33.608 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47991.log 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46222.log: gzip -5 --verbose -- /var/log/ceph/cluster1-mgr.x.log 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46222.log.gz 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47991.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47991.log.gz 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46592.log 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-mgr.x.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46957.log 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46592.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46592.log.gz 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46623.log 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46957.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46957.log.gz 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47795.log 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46623.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46623.log.gz 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47942.log 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47795.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47795.log.gz 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46739.log 2026-02-22T22:36:33.609 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47942.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47942.log.gz 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46124.log 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46739.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46739.log.gz 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46769.log 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46124.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46124.log.gz 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46059.log 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46769.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46769.log.gz 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47893.log 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46059.log: 80.2% -- replaced with /var/log/ceph/cluster1-client.admin.46059.log.gz 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47599.log 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47893.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47893.log.gz 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46591.log 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47599.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47599.log.gz 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46470.log 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46591.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46591.log.gz 2026-02-22T22:36:33.610 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.audit.log 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46470.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46470.log.gz 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47746.log 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1.audit.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46034.log 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47746.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47746.log.gz 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46908.log 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr: 89.7% -- replaced with /var/log/ceph/cluster1.audit.log.gz 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46034.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46034.log.gz 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.43024.log 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46908.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46908.log.gz 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1.log 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.43024.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.43024.log.gz 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46324.log 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.46471.log 2026-02-22T22:36:33.611 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46324.log: 85.9% -- replaced with /var/log/ceph/cluster1.log.gz 2026-02-22T22:36:33.612 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46324.log.gz 2026-02-22T22:36:33.612 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.42745.log 2026-02-22T22:36:33.612 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.46471.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.46471.log.gz 2026-02-22T22:36:33.612 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47452.log 2026-02-22T22:36:33.612 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.42745.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.42745.log.gz 2026-02-22T22:36:33.612 INFO:teuthology.orchestra.run.trial194.stderr:/var/log/ceph/cluster1-client.admin.47452.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47452.log.gz 2026-02-22T22:36:33.612 INFO:teuthology.orchestra.run.trial194.stderr: 91.4% -- replaced with /var/log/ceph/cluster2-mgr.x.log.gz 2026-02-22T22:36:33.616 INFO:teuthology.orchestra.run.trial194.stderr: 91.5% -- replaced with /var/log/ceph/cluster1-mgr.x.log.gz 2026-02-22T22:36:33.618 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50638.log 2026-02-22T22:36:33.620 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52583.log 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.50638.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48673.log 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50638.log.gz 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52583.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52583.log.gz 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46961.log 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.48673.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51002.log 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr: 64.1% -- replaced with /var/log/ceph/cluster1-client.mirror.48673.log.gz 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47645.log 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46961.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46961.log.gz 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51002.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51911.log 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47645.log: 0.0% 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.47645.log.gz 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.51002.log.gz 2026-02-22T22:36:33.621 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47841.log 2026-02-22T22:36:33.622 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51911.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51491.log 2026-02-22T22:36:33.644 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.47841.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49585.log 2026-02-22T22:36:33.644 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47841.log.gz 2026-02-22T22:36:33.644 INFO:teuthology.orchestra.run.trial198.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.51911.log.gz 2026-02-22T22:36:33.644 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.51491.log: gzip -5 --verbose -- /var/log/ceph/cluster2.log 2026-02-22T22:36:33.644 INFO:teuthology.orchestra.run.trial198.stderr: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.51491.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.49585.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49746.log 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.49585.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52077.log 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49746.log: 84.4% -- replaced with /var/log/ceph/cluster2.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48622.log 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49746.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52077.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50226.log 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50842.log 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.48622.log: 90.4% -- replaced with /var/log/ceph/cluster2-client.admin.52077.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr: 2.3% -- replaced with /var/log/ceph/cluster1-client.mirror.48622.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50226.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48554.log 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50226.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50842.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49939.log 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50842.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51636.log 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.48554.log: 70.5% -- replaced with /var/log/ceph/cluster1-client.admin.48554.log.gz 2026-02-22T22:36:33.645 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.49939.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50731.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr: 57.4% -- replaced with /var/log/ceph/cluster2-client.mirror.49939.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51636.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51294.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.50731.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49378.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51636.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr: 90.4% -- replaced with /var/log/ceph/cluster1-client.admin.50731.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51294.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48233.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51294.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49378.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49688.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49378.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48233.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.rbd-mirror-peer.49024.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48233.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49688.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52681.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49688.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.rbd-mirror-peer.49024.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49143.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52681.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50793.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52681.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49143.log: 57.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49143.log.gz 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49534.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52975.log 2026-02-22T22:36:33.646 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50793.log: /var/log/ceph/cluster2-client.admin.49534.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52779.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr: 87.1% -- replaced with /var/log/ceph/cluster2-client.admin.49534.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52975.log: gzip -5 0.0% --verbose -- -- replaced with /var/log/ceph/cluster2-client.admin.52975.log.gz /var/log/ceph/cluster2-client.admin.48776.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr: 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52779.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52877.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52779.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48776.log: 70.5% -- replaced with /var/log/ceph/cluster2-client.admin.48776.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48520.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52877.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52183.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52877.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.48520.log: 79.9% -- replaced with /var/log/ceph/cluster1-client.admin.48520.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.50670.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52183.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51867.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52183.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.50670.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46964.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr: 88.2% -- replaced with /var/log/ceph/cluster1-client.admin.50670.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51867.log: gzip -5 --verbose -- /var/log/ceph/cluster2.audit.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr: 38.7% -- replaced with /var/log/ceph/cluster2-client.mirror.51867.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46964.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46964.log.gz 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49503.log 2026-02-22T22:36:33.647 INFO:teuthology.orchestra.run.trial198.stderr:gzip/var/log/ceph/cluster2.audit.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.48086.log 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.49503.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47230.log 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.49503.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48086.log: gzip -5 --verbose 90.2% -- /var/log/ceph/cluster1-client.admin.51434.log -- replaced with /var/log/ceph/cluster2.audit.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48086.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47230.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52436.log 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47230.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 93.8% -- replaced with /var/log/ceph/cluster1-client.mirror.50793.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.51434.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47694.log 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52159.log 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 89.9% -- replaced with /var/log/ceph/cluster1-client.admin.51434.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52436.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47694.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52534.log 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47694.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 93.6% -- replaced with /var/log/ceph/cluster2-client.rbd-mirror-peer.49024.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.52159.log: gzip -5 0.0% --verbose -- -- replaced with /var/log/ceph/cluster1-client.admin.52159.log.gz /var/log/ceph/cluster2-client.mirror.51108.log 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52534.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50022.log 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52534.log.gz 2026-02-22T22:36:33.648 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51108.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50433.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51108.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.50022.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52730.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr: 83.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50022.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.50433.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52102.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.50433.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52730.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52730.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46698.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.52102.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48950.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46698.log: 90.2% -- replaced with /var/log/ceph/cluster1-client.admin.52102.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50402.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr: 79.9% -- replaced with /var/log/ceph/cluster2-client.admin.46698.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48950.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50510.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr: 66.0% -- replaced with /var/log/ceph/cluster2-client.admin.48950.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.50402.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52632.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.50510.log: 88.1% -- replaced with /var/log/ceph/cluster2-client.admin.50402.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47547.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50510.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52632.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52127.log 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52632.log.gz 2026-02-22T22:36:33.649 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.47547.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48827.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47547.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52127.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48471.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.48827.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47126.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.52127.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr: 2.3% -- replaced with /var/log/ceph/cluster2-client.mirror.48827.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.48471.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50063.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48471.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47126.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.0.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47126.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.50063.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48810.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr: 87.6% -- replaced with /var/log/ceph/cluster2-client.mirror.50063.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-osd.0.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50354.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.48810.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50193.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr: 3.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48810.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51329.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50354.log: /var/log/ceph/cluster1-client.mirror.50193.log: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50354.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49921.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51329.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49813.log 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51329.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49921.log: 57.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49921.log.gz 2026-02-22T22:36:33.650 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48639.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.49813.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46673.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr: 88.0% -- replaced with /var/log/ceph/cluster2-client.admin.49813.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49620.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.48639.log: 63.9% -- replaced with /var/log/ceph/cluster1-client.mirror.48639.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46673.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46673.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51574.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49620.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51592.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51574.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43671.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr: 57.0% -- replaced with /var/log/ceph/cluster2-client.mirror.51574.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51592.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48571.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr: 37.0% -- replaced with /var/log/ceph/cluster2-client.mirror.51592.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.43671.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43671.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43765.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.48571.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48982.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr: 55.3% -- replaced with /var/log/ceph/cluster1-client.mirror.48571.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.43765.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43765.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48742.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48982.log: gzip -5 --verbose -- /var/log/ceph/cluster2-osd.1.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr: 72.9% -- replaced with /var/log/ceph/cluster2-client.admin.48982.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48742.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52231.log 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr: 79.8% -- replaced with /var/log/ceph/cluster2-client.admin.48742.log.gz 2026-02-22T22:36:33.651 INFO:teuthology.orchestra.run.trial198.stderr: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.50193.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-osd.1.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48793.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52231.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52231.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.48656.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.48793.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50261.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr: 53.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48793.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.48656.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49781.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr: 57.7% -- replaced with /var/log/ceph/cluster1-client.mirror.48656.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50261.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51541.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50261.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49781.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52926.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49781.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51541.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49844.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52926.log: 94.0% -- replaced with /var/log/ceph/cluster1-client.mirror.49620.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.51541.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47231.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52926.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.49844.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47449.log 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47231.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.49844.log.gz 2026-02-22T22:36:33.652 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-mon.a.log 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47231.log.gz 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47449.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47449.log.gz 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48915.log 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.rbd-mirror-peer.49069.log 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-mon.a.log: /var/log/ceph/cluster1-client.admin.48915.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48895.log 65.1% 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr: -- replaced with /var/log/ceph/cluster1-client.admin.48915.log.gz 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.rbd-mirror-peer.49069.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49562.log 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.48895.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49471.log 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr: 64.1% -- replaced with /var/log/ceph/cluster2-client.mirror.48895.log.gz 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.49562.log: 87.3% -- replaced with /var/log/ceph/cluster1-client.admin.49562.log.gz 2026-02-22T22:36:33.653 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46763.log 2026-02-22T22:36:33.659 INFO:teuthology.orchestra.run.trial194.stderr: 91.3% -- replaced with /var/log/ceph/cluster1-mon.a.log.gz 2026-02-22T22:36:33.664 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49471.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50545.log 2026-02-22T22:36:33.664 INFO:teuthology.orchestra.run.trial198.stderr: 87.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49471.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46763.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47939.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46763.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.50545.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49249.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50545.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.47939.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46963.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47939.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49249.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48932.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 84.3% -- replaced with /var/log/ceph/cluster1-client.mirror.49249.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46963.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51516.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46963.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50970.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.48932.log: 63.4%/var/log/ceph/cluster2-client.admin.51516.log: -- replaced with /var/log/ceph/cluster1-client.admin.48932.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49957.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50970.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43960.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 90.3% -- replaced with /var/log/ceph/cluster2-client.admin.51516.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50970.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49957.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50041.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 83.3% -- replaced with /var/log/ceph/cluster1-client.mirror.49957.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.43960.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49024.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43960.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.50041.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46603.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.0.49024.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50475.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 87.9% -- replaced with /var/log/ceph/cluster2-client.mirror.50041.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46603.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49653.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46603.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.50475.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47743.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49653.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49177.log 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49653.log.gz 2026-02-22T22:36:33.665 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47743.log: 0.0%gzip -- replaced with /var/log/ceph/cluster2-client.admin.47743.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49976.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49177.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43510.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 56.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49177.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:gzip/var/log/ceph/cluster1-client.mirror.49976.log: -5 --verbose -- /var/log/ceph/cluster2-client.admin.47351.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.43510.log: 87.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49976.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43510.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52485.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5/var/log/ceph/cluster2-client.admin.47351.log: --verbose -- /var/log/ceph/cluster1-client.mirror.48605.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47351.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52485.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49436.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52485.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.48605.log: 3.5% -- replaced with /var/log/ceph/cluster1-client.mirror.48605.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49343.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49436.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48537.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49436.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49343.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43718.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.49343.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.48537.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48878.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 80.1% -- replaced with /var/log/ceph/cluster1-client.admin.48537.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.43718.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43718.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47988.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.48878.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.52828.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 58.0% -- replaced with /var/log/ceph/cluster2-client.mirror.48878.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.47988.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.invalid.49069.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47988.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.52828.log: gzip 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.52828.log.gz -5 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: --verbose -- /var/log/ceph/cluster2-client.mirror.48861.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 94.0% -- replaced with /var/log/ceph/cluster2-client.mirror.50475.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.invalid.49069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51143.log 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr: 64.5% -- replaced with /var/log/ceph/cluster1-client.invalid.49069.log.gz 2026-02-22T22:36:33.666 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.48861.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.49077.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 64.5% -- replaced with /var/log/ceph/cluster2-client.mirror.48861.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51143.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.49872.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.49077.log: 87.4%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.51143.log.gz -5 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: --verbose -- /var/log/ceph/cluster1-client.mirror.48588.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 57.0% -- replaced with /var/log/ceph/cluster2-client.mirror.49077.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.49872.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51979.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 87.1% -- replaced with /var/log/ceph/cluster1-client.admin.49872.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.48588.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49211.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 2.3% -- replaced with /var/log/ceph/cluster1-client.mirror.48588.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51979.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51839.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49211.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47083.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 90.0% -- replaced with /var/log/ceph/cluster2-client.admin.51979.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 84.6% -- replaced with /var/log/ceph/cluster1-client.mirror.49211.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51839.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50935.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47083.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50319.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47083.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51839.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50935.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51610.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50935.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50319.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.43892.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster1-client.mirror.50319.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51610.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48135.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.43892.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51201.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.43892.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.admin.51610.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48135.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48135.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49310.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51201.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.50756.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.51201.log.gz 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.49310.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48690.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.50756.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51075.log 2026-02-22T22:36:33.667 INFO:teuthology.orchestra.run.trial198.stderr: 93.7% -- replaced with /var/log/ceph/cluster1-client.rbd-mirror-peer.49069.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48690.log: gzip -5 --verbose 0.0% -- 90.1% /var/log/ceph/cluster2-client.mirror.0.49035.log -- replaced with /var/log/ceph/cluster2-client.admin.48690.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.50756.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51075.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.48844.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.0.49035.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51361.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: 19.6% -- replaced with /var/log/ceph/cluster2-client.mirror.0.49035.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.48844.log: 3.5%gzip -- replaced with /var/log/ceph/cluster2-client.mirror.48844.log.gz -5 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: --verbose -- /var/log/ceph/cluster2-client.admin.48759.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.51361.log: gzip -5 --verbose -- /var/log/ceph/ceph.tmp-client.admin.38311.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48759.log: 88.0% -- replaced with /var/log/ceph/cluster1-client.admin.51361.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51463.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: 80.3% -- replaced with /var/log/ceph/cluster2-client.admin.48759.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/ceph.tmp-client.admin.38311.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.51236.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/ceph.tmp-client.admin.38311.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51463.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48716.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.51236.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51463.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47498.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster2-client.mirror.51236.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48716.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48716.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47792.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.47498.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47498.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.48495.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.47792.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47792.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.51814.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.48495.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.48495.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.49998.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.51814.log: 93.9% -- replaced with /var/log/ceph/cluster1-client.mirror.49310.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47890.log 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: 93.9% -- replaced with /var/log/ceph/cluster2-client.mirror.51075.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: 93.2% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49024.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr: 90.1%/var/log/ceph/cluster1-client.mirror.49998.log: -- replaced with /var/log/ceph/cluster1-client.admin.51814.log.gz 2026-02-22T22:36:33.668 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51033.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.49998.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.47890.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47890.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.0.49015.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51033.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.49895.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.0.49015.log: 89.9% -- replaced with /var/log/ceph/cluster2-client.admin.51033.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr: 19.0% -- replaced with /var/log/ceph/cluster1-client.mirror.0.49015.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46861.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.49895.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.0.49069.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46861.log: 87.0% -- replaced with /var/log/ceph/cluster2-client.admin.49895.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46861.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.53022.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.0.49069.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47400.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.53022.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.53022.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51789.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47400.log: gzip -5 --verbose -- 0.0% /var/log/ceph/cluster1-client.admin.51400.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr: -- replaced with /var/log/ceph/cluster2-client.admin.47400.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51789.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47262.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.51400.log: 90.2% -- replaced with /var/log/ceph/cluster2-client.admin.51789.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.47596.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47262.log: 90.0% -- replaced with /var/log/ceph/cluster1-client.admin.51400.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip 0.0% -5 --verbose -- replaced with /var/log/ceph/cluster2-client.admin.47262.log.gz -- 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr: /var/log/ceph/cluster1-client.admin.50701.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.47596.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.47596.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51701.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.50701.log: gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50087.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51701.log: 90.1% -- replaced with /var/log/ceph/cluster1-client.admin.50701.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.mirror.50877.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50087.log: 90.1% -- replaced with /var/log/ceph/cluster2-client.admin.51701.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46523.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr: 84.3% -- replaced with /var/log/ceph/cluster1-client.mirror.50087.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.mirror.50877.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.46812.log 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr: 87.4% -- replaced with /var/log/ceph/cluster1-client.mirror.50877.log.gz 2026-02-22T22:36:33.669 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46523.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46523.log.gz 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50603.log 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.46812.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.46812.log.gz 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48184.log 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.mirror.50603.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.48037.log 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr: 87.5% -- replaced with /var/log/ceph/cluster2-client.mirror.50603.log.gz 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48184.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48184.log.gz 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.47084.log 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.48037.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.48037.log.gz 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-client.admin.51885.log 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.47084.log: 0.0% -- replaced with /var/log/ceph/cluster2-client.admin.47084.log.gz 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster2-osd.2.log 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-client.admin.51885.log: gzip -5 --verbose -- /var/log/ceph/cluster2-client.mirror.50108.log 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr: 90.4% -- replaced with /var/log/ceph/cluster2-client.admin.51885.log.gz 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /var/log/ceph/cluster1-client.admin.52207.log 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster2-osd.2.log: /var/log/ceph/cluster2-client.mirror.50108.log: 84.4% -- replaced with /var/log/ceph/cluster2-client.mirror.50108.log.gz 2026-02-22T22:36:33.670 INFO:teuthology.orchestra.run.trial198.stderr:/var/log/ceph/cluster1-client.admin.52207.log: 0.0% -- replaced with /var/log/ceph/cluster1-client.admin.52207.log.gz 2026-02-22T22:36:33.682 INFO:teuthology.orchestra.run.trial198.stderr: 93.2% -- replaced with /var/log/ceph/cluster2-client.mirror.0.49069.log.gz 2026-02-22T22:36:33.698 INFO:teuthology.orchestra.run.trial198.stderr: 91.3% -- replaced with /var/log/ceph/cluster2-mon.a.log.gz 2026-02-22T22:36:33.815 INFO:teuthology.orchestra.run.trial194.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-osd.2.log.gz 2026-02-22T22:36:33.855 INFO:teuthology.orchestra.run.trial198.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.0.log.gz 2026-02-22T22:36:33.863 INFO:teuthology.orchestra.run.trial194.stderr: 94.2% -- replaced with /var/log/ceph/cluster1-osd.1.log.gz 2026-02-22T22:36:33.869 INFO:teuthology.orchestra.run.trial194.stderr: 94.3% -- replaced with /var/log/ceph/cluster1-osd.0.log.gz 2026-02-22T22:36:33.871 INFO:teuthology.orchestra.run.trial194.stderr: 2026-02-22T22:36:33.872 INFO:teuthology.orchestra.run.trial194.stderr:real 0m0.287s 2026-02-22T22:36:33.872 INFO:teuthology.orchestra.run.trial194.stderr:user 0m0.828s 2026-02-22T22:36:33.872 INFO:teuthology.orchestra.run.trial194.stderr:sys 0m0.065s 2026-02-22T22:36:33.902 INFO:teuthology.orchestra.run.trial198.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.2.log.gz 2026-02-22T22:36:33.912 INFO:teuthology.orchestra.run.trial198.stderr: 94.3% -- replaced with /var/log/ceph/cluster2-osd.1.log.gz 2026-02-22T22:36:33.913 INFO:teuthology.orchestra.run.trial198.stderr: 2026-02-22T22:36:33.914 INFO:teuthology.orchestra.run.trial198.stderr:real 0m0.303s 2026-02-22T22:36:33.914 INFO:teuthology.orchestra.run.trial198.stderr:user 0m0.871s 2026-02-22T22:36:33.914 INFO:teuthology.orchestra.run.trial198.stderr:sys 0m0.123s 2026-02-22T22:36:33.915 INFO:tasks.ceph:Archiving logs... 2026-02-22T22:36:33.915 DEBUG:teuthology.misc:Transferring archived files from trial194:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242/remote/trial194/log 2026-02-22T22:36:33.917 DEBUG:teuthology.orchestra.run.trial194:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:36:34.086 DEBUG:teuthology.misc:Transferring archived files from trial198:/var/log/ceph to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242/remote/trial198/log 2026-02-22T22:36:34.086 DEBUG:teuthology.orchestra.run.trial198:> sudo tar c -f - -C /var/log/ceph -- . 2026-02-22T22:36:34.279 DEBUG:teuthology.run_tasks:Unwinding manager install 2026-02-22T22:36:34.293 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2026-02-22T22:36:34.293 DEBUG:teuthology.orchestra.run.trial194:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:36:34.295 DEBUG:teuthology.orchestra.run.trial198:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2026-02-22T22:36:39.506 DEBUG:teuthology.run_tasks:Unwinding manager clock 2026-02-22T22:36:39.520 INFO:teuthology.task.clock:Checking final clock skew... 2026-02-22T22:36:39.520 DEBUG:teuthology.orchestra.run.trial194:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:36:39.522 DEBUG:teuthology.orchestra.run.trial198:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2026-02-22T22:36:39.537 INFO:teuthology.orchestra.run.trial194.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:36:39.539 INFO:teuthology.orchestra.run.trial198.stderr:bash: line 1: ntpq: command not found 2026-02-22T22:36:39.555 INFO:teuthology.orchestra.run.trial194.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:36:39.555 INFO:teuthology.orchestra.run.trial194.stdout:=============================================================================== 2026-02-22T22:36:39.555 INFO:teuthology.orchestra.run.trial194.stdout:^* nyc3.us.ntp.li 2 6 377 38 +62us[ +98us] +/- 2637us 2026-02-22T22:36:39.555 INFO:teuthology.orchestra.run.trial194.stdout:^- chi2.us.ntp.li 2 6 377 38 -13us[ +22us] +/- 11ms 2026-02-22T22:36:39.555 INFO:teuthology.orchestra.run.trial194.stdout:^- ip216-250-115-174.pbiaas> 2 6 377 37 +1647us[+1647us] +/- 128ms 2026-02-22T22:36:39.555 INFO:teuthology.orchestra.run.trial194.stdout:^- time.ecansol.net 1 6 377 37 +3907us[+3907us] +/- 16ms 2026-02-22T22:36:39.564 INFO:teuthology.orchestra.run.trial198.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2026-02-22T22:36:39.564 INFO:teuthology.orchestra.run.trial198.stdout:=============================================================================== 2026-02-22T22:36:39.564 INFO:teuthology.orchestra.run.trial198.stdout:^+ 69-10-208-170.rainiercon> 2 6 377 102 +441us[ +593us] +/- 55ms 2026-02-22T22:36:39.564 INFO:teuthology.orchestra.run.trial198.stdout:^* ntp2.wiktel.com 1 6 377 38 -38us[ +120us] +/- 20ms 2026-02-22T22:36:39.564 INFO:teuthology.orchestra.run.trial198.stdout:^+ 172-104-28-175.ip.linode> 2 6 377 37 -4293us[-4293us] +/- 58ms 2026-02-22T22:36:39.564 INFO:teuthology.orchestra.run.trial198.stdout:^+ wdc1.us.ntp.li 3 6 377 37 +2020us[+2020us] +/- 30ms 2026-02-22T22:36:39.565 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2026-02-22T22:36:39.576 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2026-02-22T22:36:39.576 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2026-02-22T22:36:39.595 DEBUG:teuthology.orchestra.run.trial194:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T22:36:39.650 DEBUG:teuthology.orchestra.run.trial198:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2026-02-22T22:36:39.708 DEBUG:teuthology.orchestra.run.trial194:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:36:39.736 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:36:39.737 DEBUG:teuthology.orchestra.run.trial198:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 -e context=system_u:system_r:getty_t:s0 -e scontext=system_u:system_r:getty_t:s0 2026-02-22T22:36:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:36:39.764 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2026-02-22T22:36:39.780 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2026-02-22T22:36:39.791 INFO:teuthology.task.internal:Duration was 467.288883 seconds 2026-02-22T22:36:39.791 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2026-02-22T22:36:39.805 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2026-02-22T22:36:39.805 DEBUG:teuthology.orchestra.run.trial194:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:36:39.807 DEBUG:teuthology.orchestra.run.trial198:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2026-02-22T22:36:39.843 INFO:teuthology.orchestra.run.trial198.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:36:39.844 INFO:teuthology.orchestra.run.trial194.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2026-02-22T22:36:40.136 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2026-02-22T22:36:40.136 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial194.front.sepia.ceph.com 2026-02-22T22:36:40.137 DEBUG:teuthology.orchestra.run.trial194:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:36:40.156 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@trial198.front.sepia.ceph.com 2026-02-22T22:36:40.156 DEBUG:teuthology.orchestra.run.trial198:> grep -E --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | grep -E -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | grep -E -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2026-02-22T22:36:40.194 INFO:teuthology.task.internal.syslog:Gathering journactl... 2026-02-22T22:36:40.194 DEBUG:teuthology.orchestra.run.trial194:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:36:40.198 DEBUG:teuthology.orchestra.run.trial198:> sudo journalctl > /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:36:40.297 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2026-02-22T22:36:40.298 DEBUG:teuthology.orchestra.run.trial194:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:36:40.300 DEBUG:teuthology.orchestra.run.trial198:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --max-args=1 --max-procs=0 --verbose --no-run-if-empty -- gzip -5 --verbose -- 2026-02-22T22:36:40.323 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:36:40.323 INFO:teuthology.orchestra.run.trial194.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:36:40.323 INFO:teuthology.orchestra.run.trial194.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:36:40.324 INFO:teuthology.orchestra.run.trial194.stderr: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:36:40.324 INFO:teuthology.orchestra.run.trial194.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:36:40.325 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/misc.log 2026-02-22T22:36:40.325 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/kern.log 2026-02-22T22:36:40.326 INFO:teuthology.orchestra.run.trial198.stderr:/home/ubuntu/cephtest/archive/syslog/misc.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/misc.log.gz 2026-02-22T22:36:40.326 INFO:teuthology.orchestra.run.trial198.stderr:gzip -5 --verbose -- /home/ubuntu/cephtest/archive/syslog/journalctl.log 2026-02-22T22:36:40.326 INFO:teuthology.orchestra.run.trial198.stderr:/home/ubuntu/cephtest/archive/syslog/kern.log: 0.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/kern.log.gz 2026-02-22T22:36:40.333 INFO:teuthology.orchestra.run.trial194.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 81.8% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:36:40.335 INFO:teuthology.orchestra.run.trial198.stderr:/home/ubuntu/cephtest/archive/syslog/journalctl.log: 82.0% -- replaced with /home/ubuntu/cephtest/archive/syslog/journalctl.log.gz 2026-02-22T22:36:40.337 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2026-02-22T22:36:40.353 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2026-02-22T22:36:40.354 DEBUG:teuthology.orchestra.run.trial194:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:36:40.400 DEBUG:teuthology.orchestra.run.trial198:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2026-02-22T22:36:40.424 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2026-02-22T22:36:40.441 DEBUG:teuthology.orchestra.run.trial194:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:36:40.444 DEBUG:teuthology.orchestra.run.trial198:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:36:40.465 INFO:teuthology.orchestra.run.trial194.stdout:kernel.core_pattern = core 2026-02-22T22:36:40.488 INFO:teuthology.orchestra.run.trial198.stdout:kernel.core_pattern = core 2026-02-22T22:36:40.502 DEBUG:teuthology.orchestra.run.trial194:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:36:40.535 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:36:40.536 DEBUG:teuthology.orchestra.run.trial198:> test -e /home/ubuntu/cephtest/archive/coredump 2026-02-22T22:36:40.557 DEBUG:teuthology.orchestra.run:got remote process result: 1 2026-02-22T22:36:40.557 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2026-02-22T22:36:40.568 INFO:teuthology.task.internal:Transferring archived files... 2026-02-22T22:36:40.569 DEBUG:teuthology.misc:Transferring archived files from trial194:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242/remote/trial194 2026-02-22T22:36:40.569 DEBUG:teuthology.orchestra.run.trial194:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:36:40.610 DEBUG:teuthology.misc:Transferring archived files from trial198:/home/ubuntu/cephtest/archive to /home/teuthworker/mnt/teuthology/yuriw-2026-02-22_19:35:38-rbd-wip-yuri6-testing-2026-02-22-1629-squid-distro-default-trial/64242/remote/trial198 2026-02-22T22:36:40.610 DEBUG:teuthology.orchestra.run.trial198:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2026-02-22T22:36:40.640 INFO:teuthology.task.internal:Removing archive directory... 2026-02-22T22:36:40.640 DEBUG:teuthology.orchestra.run.trial194:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:36:40.644 DEBUG:teuthology.orchestra.run.trial198:> rm -rf -- /home/ubuntu/cephtest/archive 2026-02-22T22:36:40.690 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2026-02-22T22:36:40.701 INFO:teuthology.task.internal:Not uploading archives. 2026-02-22T22:36:40.701 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2026-02-22T22:36:40.719 INFO:teuthology.task.internal:Tidying up after the test... 2026-02-22T22:36:40.719 DEBUG:teuthology.orchestra.run.trial194:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:36:40.722 DEBUG:teuthology.orchestra.run.trial198:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2026-02-22T22:36:40.737 INFO:teuthology.orchestra.run.trial194.stdout: 917521 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:36 /home/ubuntu/cephtest 2026-02-22T22:36:40.747 INFO:teuthology.orchestra.run.trial198.stdout: 917572 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Feb 22 22:36 /home/ubuntu/cephtest 2026-02-22T22:36:40.748 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2026-02-22T22:36:40.766 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2026-02-22T22:36:40.816 INFO:teuthology.run:Summary data: description: rbd/mirror/{base/install clients/{mirror-extra mirror} cluster/{2-node openstack} conf/{disable-pool-app} msgr-failures/few objectstore/bluestore-low-osd-mem-target supported-random-distro$/{centos_latest} workloads/rbd-mirror-journal-bootstrap-workunit} duration: 467.28888297080994 flavor: default owner: scheduled_yuriw@soko04.front.sepia.ceph.com success: true 2026-02-22T22:36:40.816 DEBUG:teuthology.report:Pushing job info to https://paddles-paddles.apps.pok.os.sepia.ceph.com 2026-02-22T22:36:40.913 INFO:teuthology.run:pass